From 3643ce5f853b406356c7fbea015a1f6f829b6997 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Thu, 27 Sep 2018 10:58:57 -0400 Subject: [PATCH] Add publish jobs for CircleCI releases --- .circleci/config.yml | 223 +++++++++++++++++++++---------------- script/ci-release-build.js | 19 +--- 2 files changed, 130 insertions(+), 112 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b295decec16..fa11c589592f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,9 @@ env-testing-build: &env-testing-build env-release-build: &env-release-build GN_CONFIG: //electron/build/args/release.gn NOTIFY_SLACK: true + +env-publish-build: &env-publish-build + GN_CONFIG: //electron/build/args/release.gn ELECTRON_RELEASE: 1 env-browsertests: &env-browsertests @@ -44,11 +47,13 @@ env-arm: &env-arm GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' GN_EXTRA_ARGS: 'target_cpu = "arm"' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm + BUILD_NATIVE_MKSNAPSHOT: 1 env-arm64: &env-arm64 GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64 + BUILD_NATIVE_MKSNAPSHOT: 1 env-mas: &env-mas GN_EXTRA_ARGS: 'is_mas_build = true' @@ -110,9 +115,11 @@ step-setup-env-for-build: &step-setup-env-for-build # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - # https://github.com/mozilla/sccache - SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" - echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + if [ "$ELECTRON_RELEASE" != "1" ]; then + # https://github.com/mozilla/sccache + SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" + echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + fi step-install-nodejs-on-mac: &step-install-nodejs-on-mac run: @@ -128,7 +135,11 @@ step-electron-gn-gen: &step-electron-gn-gen name: Electron GN gen command: | cd src - gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS" + else + gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-electron-build: &step-electron-build run: @@ -179,10 +190,6 @@ step-persist-data-for-tests: &step-persist-data-for-tests # Build artifacts - src/out/Default/dist.zip - src/out/Default/gen/node_headers - - src/out/Default/chromedriver.zip - - src/out/Default/dsym.zip - - src/out/Default/symbols.zip - - src/out/Default/mksnapshot.zip step-electron-dist-unzip: &step-electron-dist-unzip run: @@ -198,7 +205,11 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn")'" $GN_EXTRA_ARGS" + else + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-ffmpeg-build: &step-ffmpeg-build run: @@ -214,6 +225,11 @@ step-verify-ffmpeg: &step-verify-ffmpeg cd src python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg +step-ffmpeg-store: &step-ffmpeg-store + - store_artifacts: + path: src/out/ffmpeg/ffmpeg.zip + destination: ffmpeg.zip + step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing run: name: Setup for headless testing @@ -240,18 +256,47 @@ step-mksnapshot-store: &step-mksnapshot-store path: src/out/Default/mksnapshot.zip destination: mksnapshot.zip -step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols +step-generate-breakpad_symbols: &step-generate-breakpad_symbols run: name: Generate breakpad symbols command: | - if [ "$ELECTRON_RELEASE" == "1" ]; then + cd src + # Build needed dump_syms executable + ninja -C out/Default third_party/breakpad:dump_syms + electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py + +step-native-mksnapshot-gn-gen: &step-native-mksnapshot-gn-gen + run: + name: native mksnapshot GN gen (if needed) + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then cd src - # Build needed dump_syms executable - ninja -C out/Default third_party/breakpad:dump_syms - electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" - electron/script/zip-symbols.py + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + else + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + fi + else + echo 'skipping native mksnapshot GN gen for non arm build' fi +step-native-mksnapshot-build: &step-native-mksnapshot-build + run: + name: native mksnapshot (arm/arm64) build + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then + cd src + ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip + cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip + else + echo 'skipping native mksnapshot build for non arm build' + fi + +step-native-mksnapshot-store: &step-native-mksnapshot-store + store_artifacts: + path: src/out/native_mksnapshot/native_mksnapshot.zip + destination: native_mksnapshot.zip # Lists of steps. steps-checkout: &steps-checkout @@ -315,7 +360,6 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-electron-build - *step-electron-dist-build - *step-electron-dist-store - - *step-maybe-generate-breakpad_symbols # mksnapshot - *step-mksnapshot-build @@ -336,31 +380,47 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-maybe-notify-slack-failure - *step-maybe-notify-slack-success +steps-electron-build-for-publish: &steps-electron-build-for-publish + steps: + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-gclient-sync + - *step-setup-env-for-build + + # Electron app + - *step-electron-gn-gen + - *step-electron-build + - *step-electron-dist-build + - *step-electron-dist-store + - *step-generate-breakpad_symbols + + # mksnapshot + - *step-mksnapshot-build + - *step-mksnapshot-store + + # native_mksnapshot + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store + + # chromedriver + - *step-electron-chromedriver-build + - *step-electron-chromedriver-store + + # Node.js headers + - *step-nodejs-headers-build + + steps-native-mksnapshot-build: &steps-native-mksnapshot-build steps: - attach_workspace: at: . - *step-depot-tools-add-to-path - *step-setup-env-for-build - - - run: - name: native mksnapshot GN gen - command: | - cd src - gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" - - run: - name: native mksnapshot (arm/arm64) build - command: | - cd src - ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip - cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip - - store_artifacts: - path: src/out/native_mksnapshot/native_mksnapshot.zip - destination: native_mksnapshot.zip - - persist_to_workspace: - root: . - paths: - - src/out/Default/mksnapshot.zip + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store steps-ffmpeg-build: &steps-ffmpeg-build steps: @@ -371,10 +431,8 @@ steps-ffmpeg-build: &steps-ffmpeg-build - *step-ffmpeg-gn-gen - *step-ffmpeg-build + - *step-ffmpeg-store - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.so - destination: libffmpeg.so - persist_to_workspace: root: . paths: @@ -486,9 +544,7 @@ steps-build-mac: &steps-build-mac # ffmpeg - *step-ffmpeg-gn-gen - *step-ffmpeg-build - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.dylib - destination: libffmpeg.dylib + - *step-ffmpeg-store # It would be better to verify ffmpeg as a part of a test job, # but it requires `gn` to run, and it's complicated @@ -557,6 +613,12 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-x64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-ia32-debug: <<: *machine-linux-2xlarge environment: @@ -584,6 +646,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-ia32-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-ia32 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-debug: <<: *machine-linux-2xlarge environment: @@ -611,6 +680,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-mksnapshot: <<: *machine-linux-medium environment: @@ -644,6 +720,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm64 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm64-mksnapshot: <<: *machine-linux-medium environment: @@ -786,64 +869,6 @@ workflows: requires: - linux-arm64-checkout - release-linux: - jobs: - - linux-checkout - - linux-arm-checkout - - linux-arm64-checkout - - - linux-x64-release: - requires: - - linux-checkout - - linux-x64-ffmpeg: - requires: - - linux-checkout - - linux-x64-publish: - requires: - - linux-x64-release - - linux-x64-ffmpeg - - - linux-ia32-release: - requires: - - linux-checkout - - linux-ia32-ffmpeg: - requires: - - linux-checkout - - linux-ia32-publish: - requires: - - linux-ia32-release - - linux-ia32-ffmpeg - - - linux-arm-release: - requires: - - linux-arm-checkout - - linux-arm-ffmpeg: - requires: - - linux-arm-checkout - - linux-arm-mksnapshot: - requires: - - linux-arm-checkout - - linux-arm-publish: - requires: - - linux-arm-release - - linux-arm-ffmpeg - - linux-arm-mksnapshot - - - linux-arm64-release: - requires: - - linux-arm64-checkout - - linux-arm64-ffmpeg: - requires: - - linux-arm64-checkout - - linux-arm64-mksnapshot: - requires: - - linux-arm64-checkout - - linux-arm64-publish: - requires: - - linux-arm64-release - - linux-arm64-ffmpeg - - linux-arm64-mksnapshot - build-mac-fork-prs: jobs: - osx-testing: diff --git a/script/ci-release-build.js b/script/ci-release-build.js index b78e7d97f563..42d88a9583c2 100644 --- a/script/ci-release-build.js +++ b/script/ci-release-build.js @@ -6,21 +6,15 @@ const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds' const vstsURL = 'https://github.visualstudio.com/electron/_apis/build' const appVeyorJobs = { - 'electron-x64': 'electron-n7wrc', - 'electron-ia32': 'electron-egxcs' + 'electron-x64': 'electron', + 'electron-ia32': 'electron-39ng6' } -// TODO: Enable the Build Processing preview to get the ability to trigger workflows -// programatically. -// - https://circleci.com/docs/2.0/build-processing/ - -// TODO: Update sudowoodo to somehow support monitoring workflows and individual -// builds on legacy brances const circleCIJobs = [ - 'linux-arm-release', - 'linux-arm64-release', - 'linux-ia32-release', - 'linux-x64-release' + 'linux-arm-publish', + 'linux-arm64-publish', + 'linux-ia32-publish', + 'linux-x64-publish' ] const vstsJobs = [ @@ -100,7 +94,6 @@ function buildAppVeyor (targetBranch, options) { async function callAppVeyor (targetBranch, job, options) { console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`) const environmentVariables = { - GN_CONFIG: 'release', ELECTRON_RELEASE: 1 }