chore: initial work for running releases of GN build

.circleci/config.yml

@@ -179,6 +179,10 @@ step-persist-data-for-tests: &step-persist-data-for-tests
       # Build artifacts
       - src/out/Default/dist.zip
       - src/out/Default/gen/node_headers
+      - src/out/Default/chromedriver.zip
+      - src/out/Default/dsym.zip
+      - src/out/Default/symbols.zip
+      - src/out/Default/mksnapshot.zip
 
 step-electron-dist-unzip: &step-electron-dist-unzip
   run:
@@ -201,7 +205,7 @@ step-ffmpeg-build: &step-ffmpeg-build
     name: Non proprietary ffmpeg build
     command: |
       cd src
-      ninja -C out/ffmpeg third_party/ffmpeg
+      ninja -C out/ffmpeg electron:electron_ffmpeg_zip
 
 step-verify-ffmpeg: &step-verify-ffmpeg
   run:
@@ -245,6 +249,7 @@ step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols
         # Build needed dump_syms executable
         ninja -C out/Default third_party/breakpad:dump_syms
         electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+        electron/script/zip-symbols.py
       fi
 
 
@@ -352,6 +357,10 @@ steps-native-mksnapshot-build: &steps-native-mksnapshot-build
     - store_artifacts:
         path: src/out/native_mksnapshot/native_mksnapshot.zip
         destination: native_mksnapshot.zip
+    - persist_to_workspace:
+        root: .
+        paths:
+          - src/out/Default/mksnapshot.zip
 
 steps-ffmpeg-build: &steps-ffmpeg-build
   steps:
@@ -370,6 +379,7 @@ steps-ffmpeg-build: &steps-ffmpeg-build
         root: .
         paths:
           - src/out/ffmpeg/libffmpeg.so
+          - src/out/ffmpeg/ffmpeg.zip
 
 steps-native-tests: &steps-native-tests
   steps:
@@ -776,6 +786,64 @@ workflows:
           requires:
             - linux-arm64-checkout
 
+  release-linux:
+    jobs:
+      - linux-checkout
+      - linux-arm-checkout
+      - linux-arm64-checkout
+
+      - linux-x64-release:
+          requires:
+            - linux-checkout
+      - linux-x64-ffmpeg:
+          requires:
+            - linux-checkout
+      - linux-x64-publish:
+          requires:
+            - linux-x64-release
+            - linux-x64-ffmpeg
+
+      - linux-ia32-release:
+          requires:
+            - linux-checkout
+      - linux-ia32-ffmpeg:
+          requires:
+            - linux-checkout
+      - linux-ia32-publish:
+          requires:
+            - linux-ia32-release
+            - linux-ia32-ffmpeg
+
+      - linux-arm-release:
+          requires:
+            - linux-arm-checkout
+      - linux-arm-ffmpeg:
+           requires:
+             - linux-arm-checkout
+      - linux-arm-mksnapshot:
+          requires:
+            - linux-arm-checkout
+      - linux-arm-publish:
+          requires:
+            - linux-arm-release
+            - linux-arm-ffmpeg
+            - linux-arm-mksnapshot
+
+      - linux-arm64-release:
+          requires:
+            - linux-arm64-checkout
+      - linux-arm64-ffmpeg:
+           requires:
+             - linux-arm64-checkout
+      - linux-arm64-mksnapshot:
+          requires:
+            - linux-arm64-checkout
+      - linux-arm64-publish:
+          requires:
+            - linux-arm64-release
+            - linux-arm64-ffmpeg
+            - linux-arm64-mksnapshot
+
   build-mac-fork-prs:
     jobs:
       - osx-testing:

appveyor.yml

@@ -38,6 +38,7 @@ build_script:
   - ps: >-
       if ($env:GN_CONFIG -eq 'release') {
         python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms
+        python electron\script\zip-symbols.py
         appveyor PushArtifact out/Default/electron.breakpad.syms
       }
 test_script:
@@ -52,3 +53,15 @@ test_script:
       }
   - cd electron
   - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci )
+deploy_script:
+  - cd electron
+  - ps: >-
+      if (Test-Path Env:\ELECTRON_RELEASE) {
+        if (Test-Path Env:\UPLOAD_TO_S3) {
+          Write-Output "Uploading Electron release distribution to s3"
+          & python script\upload.py --upload_to_s3
+        } else {
+          Write-Output "Uploading Electron release distribution to github releases"
+          & python script\upload.py
+        }
+      }

script/ci-release-build.js

@@ -6,16 +6,21 @@ const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds'
 const vstsURL = 'https://github.visualstudio.com/electron/_apis/build'
 
 const appVeyorJobs = {
-  'electron-x64': 'electron',
+  'electron-x64': 'electron-n7wrc',
-  'electron-ia32': 'electron-39ng6'
+  'electron-ia32': 'electron-egxcs'
 }
 
+// TODO: Enable the Build Processing preview to get the ability to trigger workflows
+//       programatically.
+//       - https://circleci.com/docs/2.0/build-processing/
+
+// TODO: Update sudowoodo to somehow support monitoring workflows and individual
+//       builds on legacy brances
 const circleCIJobs = [
-  'electron-linux-arm',
+  'linux-arm-release',
-  'electron-linux-arm64',
+  'linux-arm64-release',
-  'electron-linux-ia32',
+  'linux-ia32-release',
-  //  'electron-linux-mips64el',
+  'linux-x64-release'
-  'electron-linux-x64'
 ]
 
 const vstsJobs = [
@@ -94,16 +99,13 @@ function buildAppVeyor (targetBranch, options) {
 
 async function callAppVeyor (targetBranch, job, options) {
   console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`)
-  let environmentVariables = {}
+  const environmentVariables = {
-
+    GN_CONFIG: 'release',
-  if (options.ghRelease) {
+    ELECTRON_RELEASE: 1
-    environmentVariables.ELECTRON_RELEASE = 1
-  } else {
-    environmentVariables.RUN_RELEASE_BUILD = 'true'
   }
 
-  if (options.automaticRelease) {
+  if (!options.ghRelease) {
-    environmentVariables.AUTO_RELEASE = 'true'
+    environmentVariables.UPLOAD_TO_S3 = 1
   }
 
   const requestOpts = {
@@ -144,16 +146,14 @@ async function buildVSTS (targetBranch, options) {
     assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`)
   }
   console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`)
-  let environmentVariables = {}
+  const environmentVariables = {
+    ELECTRON_RELEASE: 1
+  }
 
   if (!options.ghRelease) {
     environmentVariables.UPLOAD_TO_S3 = 1
   }
 
-  if (options.automaticRelease) {
-    environmentVariables.AUTO_RELEASE = 'true'
-  }
-
   let requestOpts = {
     url: `${vstsURL}/definitions?api-version=4.1`,
     auth: {

script/dump-symbols.py

@@ -9,7 +9,7 @@ from lib.util import get_electron_branding, execute, rm_rf
 
 ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT))
-RELEASE_PATH = os.path.join('out', 'Release')
+RELEASE_PATH = os.path.join('out', 'Default')
 
 def main():
   args = parse_args()

script/lib/util.py

@@ -21,6 +21,7 @@ import zipfile
 from lib.config import is_verbose_mode, PLATFORM
 from lib.env_util import get_vs_env
 
+GN_SRC_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..'))
 BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor',
                                         'boto'))
 
@@ -295,3 +296,10 @@ def make_version(major, minor, patch, pre = None):
     return major + '.' + minor + '.' + patch
   return major + "." + minor + "." + patch + '-' + pre
 
+def get_out_dir():
+  out_dir = 'Debug'
+  override = os.environ.get('ELECTRON_OUT_DIR')
+  if override is not None:
+    out_dir = override
+  return os.path.join(GN_SRC_DIR, 'out', out_dir)
+

script/upload-index-json.py

@@ -7,6 +7,8 @@ import urllib2
 from lib.config import s3_config
 from lib.util import s3put, scoped_cwd, safe_mkdir
 
+# TODO: Update this entire file to point at the correct file names in the out
+#       directory
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 OUT_DIR     = os.path.join(SOURCE_ROOT, 'out', 'D')
 

script/upload-node-checksums.py

@@ -10,7 +10,8 @@ import tempfile
 from lib.config import s3_config
 from lib.util import download, rm_rf, s3put, safe_mkdir
 
-
+# TODO: Update this entire file to point at the correct file names in the out
+#       directory
 DIST_URL = 'https://atom.io/download/electron/'
 
 

script/upload-node-headers.py

@@ -9,7 +9,8 @@ import sys
 from lib.config import PLATFORM, get_target_arch, s3_config
 from lib.util import safe_mkdir, scoped_cwd, s3put
 
-
+# TODO: Update this entire file to point at the correct file names in the out
+#       directory
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 DIST_DIR    = os.path.join(SOURCE_ROOT, 'dist')
 OUT_DIR     = os.path.join(SOURCE_ROOT, 'out', 'R')

script/upload-symbols.py

@@ -7,7 +7,8 @@ import sys
 from lib.config import PLATFORM, s3_config, enable_verbose_mode
 from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put
 
-
+# TODO: Update this entire file to point at the correct file names in the out
+#       directory
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
 RELEASE_DIR = os.path.join(SOURCE_ROOT, 'out', 'R')

script/upload.py

@@ -33,6 +33,8 @@ DSYM_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'dsym')
 PDB_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'pdb')
 
 
+# TODO: Update this entire file to point at the correct file names in the out
+#       directory
 def main():
   args = parse_args()
   if  args.upload_to_s3:

script/zip-symbols.py

@@ -0,0 +1,37 @@
+import glob
+import os
+import sys
+
+from lib.config import PLATFORM, get_target_arch
+from lib.util import scoped_cwd, get_electron_version, make_zip, \
+                     electron_gyp, get_out_dir
+
+ELECTRON_VERSION = get_electron_version()
+PROJECT_NAME = electron_gyp()['project_name%']
+OUT_DIR = get_out_dir()
+
+def main():
+  if get_target_arch() == 'mips64el':
+    return
+
+  dist_name = 'symbols.zip'
+  zip_file = os.path.join(OUT_DIR, dist_name)
+  licenses = ['LICENSE', 'LICENSES.chromium.html', 'version']
+
+  with scoped_cwd(OUT_DIR):
+    dirs = ['{0}.breakpad.syms'.format(PROJECT_NAME)]
+    make_zip(zip_file, licenses, dirs)
+
+  if PLATFORM == 'darwin':
+    dsym_name = 'dsym.zip'
+    with scoped_cwd(OUT_DIR):
+      dsyms = glob.glob('*.dSYM')
+      make_zip(os.path.join(OUT_DIR, dsym_name), licenses, dsyms)
+  elif PLATFORM == 'win32':
+    pdb_name = 'pdb.zip'
+    with scoped_cwd(OUT_DIR):
+      pdbs = glob.glob('*.pdb')
+      make_zip(os.path.join(OUT_DIR, pdb_name), pdbs + licenses, [])
+
+if __name__ == '__main__':
+  sys.exit(main())

vsts.yml

@@ -61,7 +61,7 @@ jobs:
   - bash: |
       cd src
       gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-      ninja -C out/ffmpeg third_party/ffmpeg
+      ninja -C out/ffmpeg electron:electron_ffmpeg_zip
     displayName: Non proprietary ffmpeg build
     condition: and(succeeded(), eq(variables['RUN_TESTS'], '1'))
 
@@ -70,6 +70,7 @@ jobs:
       # Build needed dump_syms executable
       ninja -C out/Default third_party/breakpad:dump_syms
       electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
+      electron/script/zip-symbols.py
     displayName: Ninja build app
     condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
 
@@ -115,6 +116,18 @@ jobs:
       ninja -C out/Default electron:electron_mksnapshot_zip
     displayName: Build mksnapshot and zip
 
+  - bash: |
+      cd src/electron
+      if [ "$UPLOAD_TO_S3" != "1" ]; then
+        echo 'Uploading Electron release distribution to github releases'
+        ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py
+      else
+        echo 'Uploading Electron release distribution to s3'
+        ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py --upload_to_s3
+      fi
+    name: Upload_distribution
+    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
+
   - task: PublishTestResults@2
     displayName: Publish Test Results
     inputs: