Merge pull request #11001 from electron/ci-release-build-api

Decouple building electron for release from the actual release cutting process

.circleci/config.yml

@@ -19,6 +19,10 @@ jobs:
             case ${MESSAGE} in
               Bump* ) echo 'export ELECTRON_RELEASE=1' >> $BASH_ENV
             esac
+            if [ -n "${RUN_RELEASE_BUILD}" ]; then
+              echo 'release build triggered from api'
+              echo 'export ELECTRON_RELEASE=1 TRIGGERED_BY_API=1' >> $BASH_ENV
+            fi
       - run:
          name: Bootstrap
          command: |
@@ -52,9 +56,12 @@ jobs:
       - run:
           name: Upload distribution
           command: |
-             if [ "$ELECTRON_RELEASE" == "1" ]; then
+             if [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" != "1" ]; then
-                echo 'Uploading Electron release distribution'
+                echo 'Uploading Electron release distribution to github releases'
                 script/upload.py
+             elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" == "1" ]; then
+                echo 'Uploading Electron release distribution to s3'
+                script/upload.py --upload_to_s3
              else
                 echo 'Skipping upload distribution because build is not for release'
              fi
@@ -63,7 +70,7 @@ jobs:
       - image: electronbuilds/electron:0.0.3
         environment:
           TARGET_ARCH: arm64
-    resource_class: xlarge          
+    resource_class: xlarge
     steps:
       - checkout
       - run:
@@ -76,6 +83,10 @@ jobs:
             case ${MESSAGE} in
               Bump* ) echo 'export ELECTRON_RELEASE=1' >> $BASH_ENV
             esac
+            if [ -n "${RUN_RELEASE_BUILD}" ]; then
+              echo 'release build triggered from api'
+              echo 'export ELECTRON_RELEASE=1 TRIGGERED_BY_API=1' >> $BASH_ENV
+            fi
       - run:
          name: Bootstrap
          command: |
@@ -109,9 +120,12 @@ jobs:
       - run:
           name: Upload distribution
           command: |
-             if [ "$ELECTRON_RELEASE" == "1" ]; then
+             if [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" != "1" ]; then
-                echo 'Uploading Electron release distribution'
+                echo 'Uploading Electron release distribution to github releases'
                 script/upload.py
+             elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" == "1" ]; then
+                echo 'Uploading Electron release distribution to s3'
+                script/upload.py --upload_to_s3
              else
                 echo 'Skipping upload distribution because build is not for release'
              fi
@@ -133,6 +147,10 @@ jobs:
             case ${MESSAGE} in
               Bump* ) echo 'export ELECTRON_RELEASE=1' >> $BASH_ENV
             esac
+            if [ -n "${RUN_RELEASE_BUILD}" ]; then
+              echo 'release build triggered from api'
+              echo 'export ELECTRON_RELEASE=1 TRIGGERED_BY_API=1' >> $BASH_ENV
+            fi
       - run:
          name: Bootstrap
          command: |
@@ -166,9 +184,12 @@ jobs:
       - run:
           name: Upload distribution
           command: |
-             if [ "$ELECTRON_RELEASE" == "1" ]; then
+             if [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" != "1" ]; then
-                echo 'Uploading Electron release distribution'
+                echo 'Uploading Electron release distribution to github releases'
                 script/upload.py
+             elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" == "1" ]; then
+                echo 'Uploading Electron release distribution to s3'
+                script/upload.py --upload_to_s3
              else
                 echo 'Skipping upload distribution because build is not for release'
              fi
@@ -191,6 +212,10 @@ jobs:
             case ${MESSAGE} in
               Bump* ) echo 'export ELECTRON_RELEASE=1' >> $BASH_ENV
             esac
+            if [ -n "${RUN_RELEASE_BUILD}" ]; then
+              echo 'release build triggered from api'
+              echo 'export ELECTRON_RELEASE=1 TRIGGERED_BY_API=1' >> $BASH_ENV
+            fi
       - run:
          name: Bootstrap
          command: |
@@ -224,9 +249,12 @@ jobs:
       - run:
           name: Upload distribution
           command: |
-             if [ "$ELECTRON_RELEASE" == "1" ]; then
+             if [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" != "1" ]; then
-                echo 'Uploading Electron release distribution'
+                echo 'Uploading Electron release distribution to github releases'
                 script/upload.py
+             elif [ "$ELECTRON_RELEASE" == "1" ] && [ "$TRIGGERED_BY_API" == "1" ]; then
+                echo 'Uploading Electron release distribution to s3'
+                script/upload.py --upload_to_s3
              else
                 echo 'Skipping upload distribution because build is not for release'
              fi

package.json

@@ -53,6 +53,7 @@
     "create-api-json": "electron-docs-linter docs --outfile=out/electron-api.json --version=$npm_package_version",
     "create-typescript-definitions": "npm run create-api-json && electron-typescript-definitions --in=out/electron-api.json --out=out/electron.d.ts",
     "merge-release": "node ./script/merge-release.js",
+    "mock-release": "node ./script/ci-release-build.js",
     "preinstall": "node -e 'process.exit(0)'",
     "publish-to-npm": "node ./script/publish-to-npm.js",
     "prepack": "check-for-leaks",

script/ci-release-build.js

@@ -0,0 +1,56 @@
+const args = require('minimist')(process.argv.slice(2))
+const assert = require('assert')
+const request = require('request')
+
+const ciJobs = [
+  'electron-linux-arm64',
+  'electron-linux-ia32',
+  'electron-linux-x64',
+  'electron-linux-arm'
+]
+
+const CIcall = (buildUrl, targetBranch, job) => {
+  console.log(`Triggering CircleCI to run build job: ${job} on branch: ${targetBranch} with release flag.`)
+
+  request({
+    method: 'POST',
+    url: buildUrl,
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': 'application/json'
+    },
+    body: JSON.stringify({
+      'build_parameters': {
+        'RUN_RELEASE_BUILD': 'true',
+        'CIRCLE_JOB': job
+      }
+    })
+  }, (err, res, body) => {
+    if (!err && res.statusCode >= 200 && res.statusCode < 300) {
+      const build = JSON.parse(body)
+      console.log(`Check ${build.build_url} for status. (${job})`)
+    } else {
+      console.log('Error: ', `(status ${res.statusCode})`, err || JSON.parse(res.body), job)
+    }
+  })
+}
+
+if (args._.length < 1) {
+  console.log(`Trigger Circle CI to build release builds of electron.
+  Usage: ci-release-build.js [--job=CI_JOB_NAME] TARGET_BRANCH
+  `)
+  process.exit(0)
+}
+
+assert(process.env.CIRCLE_TOKEN, 'CIRCLE_TOKEN not found in environment')
+
+const targetBranch = args._[0]
+const job = args['job']
+const circleBuildUrl = `https://circleci.com/api/v1.1/project/github/electron/electron/tree/${targetBranch}?circle-token=${process.env.CIRCLE_TOKEN}`
+
+if (job) {
+  assert(ciJobs.includes(job), `Unknown CI job name: ${job}.`)
+  CIcall(circleBuildUrl, targetBranch, job)
+} else {
+  ciJobs.forEach((job) => CIcall(circleBuildUrl, targetBranch, job))
+}

script/upload.py

@@ -56,36 +56,44 @@ def main():
       tag_exists = True
       break
 
-  assert tag_exists == args.overwrite, \
+  if not args.upload_to_s3:
-         'You have to pass --overwrite to overwrite a published release'
+    assert tag_exists == args.overwrite, \
-
+          'You have to pass --overwrite to overwrite a published release'
-  if not args.overwrite:
+    if not args.overwrite:
-    release = create_or_get_release_draft(github, releases, args.version,
+      release = create_or_get_release_draft(github, releases, args.version,
-                                          tag_exists)
+                                            tag_exists)
 
   # Upload Electron with GitHub Releases API.
-  upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME))
+  upload_electron(github, release, os.path.join(DIST_DIR, DIST_NAME),
-  upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME))
+                  args.upload_to_s3)
+  upload_electron(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME),
+                  args.upload_to_s3)
   if PLATFORM == 'darwin':
     upload_electron(github, release, os.path.join(DIST_DIR,
-                    'electron-api.json'))
+                    'electron-api.json'), args.upload_to_s3)
-    upload_electron(github, release, os.path.join(DIST_DIR, 'electron.d.ts'))
+    upload_electron(github, release, os.path.join(DIST_DIR, 'electron.d.ts'),
-    upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME))
+                    args.upload_to_s3)
+    upload_electron(github, release, os.path.join(DIST_DIR, DSYM_NAME),
+                    args.upload_to_s3)
   elif PLATFORM == 'win32':
-    upload_electron(github, release, os.path.join(DIST_DIR, PDB_NAME))
+    upload_electron(github, release, os.path.join(DIST_DIR, PDB_NAME),
+                    args.upload_to_s3)
 
   # Upload free version of ffmpeg.
   ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION)
-  upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg))
+  upload_electron(github, release, os.path.join(DIST_DIR, ffmpeg),
+                  args.upload_to_s3)
 
   # Upload chromedriver and mksnapshot for minor version update.
   if parse_version(args.version)[2] == '0':
     chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION)
-    upload_electron(github, release, os.path.join(DIST_DIR, chromedriver))
+    upload_electron(github, release, os.path.join(DIST_DIR, chromedriver),
+                    args.upload_to_s3)
     mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
-    upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot))
+    upload_electron(github, release, os.path.join(DIST_DIR, mksnapshot),
+                    args.upload_to_s3)
 
-  if PLATFORM == 'win32' and not tag_exists:
+  if PLATFORM == 'win32' and not tag_exists and not args.upload_to_s3:
     # Upload PDBs to Windows symbol server.
     run_python_script('upload-windows-pdb.py')
 
@@ -104,6 +112,12 @@ def parse_args():
   parser.add_argument('-p', '--publish-release',
                       help='Publish the release',
                       action='store_true')
+  parser.add_argument('-s', '--upload_to_s3',
+                      help='Upload assets to s3 bucket',
+                      dest='upload_to_s3',
+                      action='store_true',
+                      default=False,
+                      required=False)
   return parser.parse_args()
 
 
@@ -187,7 +201,17 @@ def create_release_draft(github, tag):
   return r
 
 
-def upload_electron(github, release, file_path):
+def upload_electron(github, release, file_path, upload_to_s3):
+
+  # if upload_to_s3 is set, skip github upload.
+  if upload_to_s3:
+    bucket, access_key, secret_key = s3_config()
+    key_prefix = 'electron-artifacts/{0}'.format(release['tag_name'])
+    s3put(bucket, access_key, secret_key, os.path.dirname(file_path),
+          key_prefix, [file_path])
+    upload_sha256_checksum(release['tag_name'], file_path, key_prefix)
+    return
+
   # Delete the original file before uploading in CI.
   filename = os.path.basename(file_path)
   if os.environ.has_key('CI'):
@@ -210,7 +234,7 @@ def upload_electron(github, release, file_path):
     arm_filename = filename.replace('armv7l', 'arm')
     arm_file_path = os.path.join(os.path.dirname(file_path), arm_filename)
     shutil.copy2(file_path, arm_file_path)
-    upload_electron(github, release, arm_file_path)
+    upload_electron(github, release, arm_file_path, upload_to_s3)
 
 
 def upload_io_to_github(release, filename, filepath):
@@ -220,9 +244,11 @@ def upload_io_to_github(release, filename, filepath):
   execute(['node', script_path, filepath, filename, str(release['id'])])
 
 
-def upload_sha256_checksum(version, file_path):
+def upload_sha256_checksum(version, file_path, key_prefix=None):
   bucket, access_key, secret_key = s3_config()
   checksum_path = '{}.sha256sum'.format(file_path)
+  if key_prefix is None:
+    key_prefix = 'atom-shell/tmp/{0}'.format(version)
   sha256 = hashlib.sha256()
   with open(file_path, 'rb') as f:
     sha256.update(f.read())
@@ -231,7 +257,7 @@ def upload_sha256_checksum(version, file_path):
   with open(checksum_path, 'w') as checksum:
     checksum.write('{} *{}'.format(sha256.hexdigest(), filename))
   s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path),
-        'atom-shell/tmp/{0}'.format(version), [checksum_path])
+        key_prefix, [checksum_path])
 
 
 def auth_token():