diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ab4259d9ad1..701f6b169ed2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,9 @@ env-testing-build: &env-testing-build env-release-build: &env-release-build GN_CONFIG: //electron/build/args/release.gn NOTIFY_SLACK: true + +env-publish-build: &env-publish-build + GN_CONFIG: //electron/build/args/release.gn ELECTRON_RELEASE: 1 env-browsertests: &env-browsertests @@ -44,11 +47,13 @@ env-arm: &env-arm GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' GN_EXTRA_ARGS: 'target_cpu = "arm"' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm + BUILD_NATIVE_MKSNAPSHOT: 1 env-arm64: &env-arm64 GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64 + BUILD_NATIVE_MKSNAPSHOT: 1 env-mas: &env-mas GN_EXTRA_ARGS: 'is_mas_build = true' @@ -110,9 +115,11 @@ step-setup-env-for-build: &step-setup-env-for-build # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - # https://github.com/mozilla/sccache - SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" - echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + if [ "$ELECTRON_RELEASE" != "1" ]; then + # https://github.com/mozilla/sccache + SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" + echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + fi step-install-nodejs-on-mac: &step-install-nodejs-on-mac run: @@ -128,7 +135,11 @@ step-electron-gn-gen: &step-electron-gn-gen name: Electron GN gen command: | cd src - gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS" + else + gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-electron-build: &step-electron-build run: @@ -194,14 +205,18 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn")'" $GN_EXTRA_ARGS" + else + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-ffmpeg-build: &step-ffmpeg-build run: name: Non proprietary ffmpeg build command: | cd src - ninja -C out/ffmpeg third_party/ffmpeg + ninja -C out/ffmpeg electron:electron_ffmpeg_zip step-verify-ffmpeg: &step-verify-ffmpeg run: @@ -210,6 +225,11 @@ step-verify-ffmpeg: &step-verify-ffmpeg cd src python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg +step-ffmpeg-store: &step-ffmpeg-store + store_artifacts: + path: src/out/ffmpeg/ffmpeg.zip + destination: ffmpeg.zip + step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing run: name: Setup for headless testing @@ -236,17 +256,47 @@ step-mksnapshot-store: &step-mksnapshot-store path: src/out/Default/mksnapshot.zip destination: mksnapshot.zip -step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols +step-generate-breakpad_symbols: &step-generate-breakpad_symbols run: name: Generate breakpad symbols command: | - if [ "$ELECTRON_RELEASE" == "1" ]; then + cd src + # Build needed dump_syms executable + ninja -C out/Default third_party/breakpad:dump_syms + electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py + +step-native-mksnapshot-gn-gen: &step-native-mksnapshot-gn-gen + run: + name: native mksnapshot GN gen (if needed) + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then cd src - # Build needed dump_syms executable - ninja -C out/Default third_party/breakpad:dump_syms - electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + else + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + fi + else + echo 'skipping native mksnapshot GN gen for non arm build' fi +step-native-mksnapshot-build: &step-native-mksnapshot-build + run: + name: native mksnapshot (arm/arm64) build + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then + cd src + ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip + cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip + else + echo 'skipping native mksnapshot build for non arm build' + fi + +step-native-mksnapshot-store: &step-native-mksnapshot-store + store_artifacts: + path: src/out/native_mksnapshot/native_mksnapshot.zip + destination: native_mksnapshot.zip # Lists of steps. steps-checkout: &steps-checkout @@ -310,7 +360,6 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-electron-build - *step-electron-dist-build - *step-electron-dist-store - - *step-maybe-generate-breakpad_symbols # mksnapshot - *step-mksnapshot-build @@ -331,27 +380,47 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-maybe-notify-slack-failure - *step-maybe-notify-slack-success +steps-electron-build-for-publish: &steps-electron-build-for-publish + steps: + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-gclient-sync + - *step-setup-env-for-build + + # Electron app + - *step-electron-gn-gen + - *step-electron-build + - *step-electron-dist-build + - *step-electron-dist-store + - *step-generate-breakpad_symbols + + # mksnapshot + - *step-mksnapshot-build + - *step-mksnapshot-store + + # native_mksnapshot + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store + + # chromedriver + - *step-electron-chromedriver-build + - *step-electron-chromedriver-store + + # Node.js headers + - *step-nodejs-headers-build + + steps-native-mksnapshot-build: &steps-native-mksnapshot-build steps: - attach_workspace: at: . - *step-depot-tools-add-to-path - *step-setup-env-for-build - - - run: - name: native mksnapshot GN gen - command: | - cd src - gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" - - run: - name: native mksnapshot (arm/arm64) build - command: | - cd src - ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip - cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip - - store_artifacts: - path: src/out/native_mksnapshot/native_mksnapshot.zip - destination: native_mksnapshot.zip + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store steps-ffmpeg-build: &steps-ffmpeg-build steps: @@ -362,14 +431,13 @@ steps-ffmpeg-build: &steps-ffmpeg-build - *step-ffmpeg-gn-gen - *step-ffmpeg-build + - *step-ffmpeg-store - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.so - destination: libffmpeg.so - persist_to_workspace: root: . paths: - src/out/ffmpeg/libffmpeg.so + - src/out/ffmpeg/ffmpeg.zip steps-native-tests: &steps-native-tests steps: @@ -476,9 +544,7 @@ steps-build-mac: &steps-build-mac # ffmpeg - *step-ffmpeg-gn-gen - *step-ffmpeg-build - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.dylib - destination: libffmpeg.dylib + - *step-ffmpeg-store # It would be better to verify ffmpeg as a part of a test job, # but it requires `gn` to run, and it's complicated @@ -547,6 +613,12 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-x64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-ia32-debug: <<: *machine-linux-2xlarge environment: @@ -574,6 +646,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-ia32-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-ia32 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-debug: <<: *machine-linux-2xlarge environment: @@ -601,6 +680,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-mksnapshot: <<: *machine-linux-medium environment: @@ -634,6 +720,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm64 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm64-mksnapshot: <<: *machine-linux-medium environment: diff --git a/DEPS b/DEPS index 029e9db03943..19fcb02031e8 100644 --- a/DEPS +++ b/DEPS @@ -65,6 +65,33 @@ hooks = [ 'pattern': 'src/electron/package.json', 'name': 'electron_npm_deps' }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.system("git submodule update --init --recursive");', + ], + 'pattern': 'src/electron', + 'name': 'electron_submodules' + }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("boto"); os.system("python setup.py build");', + ], + 'pattern': 'src/electron', + 'name': 'setup_boto', + }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("requests"); os.system("python setup.py build");', + ], + 'pattern': 'src/electron', + 'name': 'setup_requests', + } ] recursedeps = [ diff --git a/appveyor.yml b/appveyor.yml index d0871e5cdec0..9f3542a5a6f5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,6 +38,7 @@ build_script: - ps: >- if ($env:GN_CONFIG -eq 'release') { python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms + python electron\script\zip-symbols.py appveyor PushArtifact out/Default/electron.breakpad.syms } test_script: @@ -52,3 +53,16 @@ test_script: } - cd electron - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci ) + - cd .. +deploy_script: + - cd electron + - ps: >- + if (Test-Path Env:\ELECTRON_RELEASE) { + if (Test-Path Env:\UPLOAD_TO_S3) { + Write-Output "Uploading Electron release distribution to s3" + & python script\upload.py --upload_to_s3 + } else { + Write-Output "Uploading Electron release distribution to github releases" + & python script\upload.py + } + } \ No newline at end of file diff --git a/docs/development/upgrading-chromium.md b/docs/development/upgrading-chromium.md index 0f6fb66d7f6a..bf75a36701cf 100644 --- a/docs/development/upgrading-chromium.md +++ b/docs/development/upgrading-chromium.md @@ -97,7 +97,7 @@ This is an overview of the steps needed to upgrade Chromium in Electron. - Add more build flags to disable features in build-time. When a Debug build of Electron succeeds, run the tests: -`$ ./script/test.py` +`$ npm run test` Fix the failing tests. Follow all the steps above to fix Electron code on all supported platforms. diff --git a/script/ci-release-build.js b/script/ci-release-build.js index 04c8cb5a7dde..42d88a9583c2 100644 --- a/script/ci-release-build.js +++ b/script/ci-release-build.js @@ -11,11 +11,10 @@ const appVeyorJobs = { } const circleCIJobs = [ - 'electron-linux-arm', - 'electron-linux-arm64', - 'electron-linux-ia32', - // 'electron-linux-mips64el', - 'electron-linux-x64' + 'linux-arm-publish', + 'linux-arm64-publish', + 'linux-ia32-publish', + 'linux-x64-publish' ] const vstsJobs = [ @@ -94,16 +93,12 @@ function buildAppVeyor (targetBranch, options) { async function callAppVeyor (targetBranch, job, options) { console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`) - let environmentVariables = {} - - if (options.ghRelease) { - environmentVariables.ELECTRON_RELEASE = 1 - } else { - environmentVariables.RUN_RELEASE_BUILD = 'true' + const environmentVariables = { + ELECTRON_RELEASE: 1 } - if (options.automaticRelease) { - environmentVariables.AUTO_RELEASE = 'true' + if (!options.ghRelease) { + environmentVariables.UPLOAD_TO_S3 = 1 } const requestOpts = { @@ -144,16 +139,14 @@ async function buildVSTS (targetBranch, options) { assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`) } console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`) - let environmentVariables = {} + const environmentVariables = { + ELECTRON_RELEASE: 1 + } if (!options.ghRelease) { environmentVariables.UPLOAD_TO_S3 = 1 } - if (options.automaticRelease) { - environmentVariables.AUTO_RELEASE = 'true' - } - let requestOpts = { url: `${vstsURL}/definitions?api-version=4.1`, auth: { diff --git a/script/dump-symbols.py b/script/dump-symbols.py index c4884e40b92e..f8f4597d31fe 100755 --- a/script/dump-symbols.py +++ b/script/dump-symbols.py @@ -5,11 +5,11 @@ import os import sys from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode -from lib.util import get_electron_branding, execute, rm_rf +from lib.util import get_electron_branding, execute, rm_rf, get_out_dir ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT)) -RELEASE_PATH = os.path.join('out', 'Release') +RELEASE_PATH = get_out_dir() def main(): args = parse_args() diff --git a/script/lib/util.py b/script/lib/util.py index d82bd6a11255..26593413976b 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -21,6 +21,7 @@ import zipfile from lib.config import is_verbose_mode, PLATFORM from lib.env_util import get_vs_env +GN_SRC_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..')) BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'boto')) @@ -295,3 +296,22 @@ def make_version(major, minor, patch, pre = None): return major + '.' + minor + '.' + patch return major + "." + minor + "." + patch + '-' + pre +def get_out_dir(): + out_dir = 'Debug' + override = os.environ.get('ELECTRON_OUT_DIR') + if override is not None: + out_dir = override + return os.path.join(GN_SRC_DIR, 'out', out_dir) + +# NOTE: This path is not created by gn, it is used as a scratch zone by our +# upload scripts +def get_dist_dir(): + return os.path.join(get_out_dir(), 'gen', 'electron_dist') + +def get_electron_exec(): + if sys.platform == 'darwin': + return 'out/{0}/Electron.app/Contents/MacOS/Electron'.format(get_out_dir()) + elif sys.platform == 'win32': + return 'out/{0}/electron.exe'.format(get_out_dir()) + elif sys.platform == 'linux': + return 'out/{0}/electron'.format(get_out_dir()) \ No newline at end of file diff --git a/script/rebuild-test-modules.py b/script/rebuild-test-modules.py deleted file mode 100755 index abf1800e33ce..000000000000 --- a/script/rebuild-test-modules.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -import argparse -import os -import shutil -import subprocess -import sys - -from lib.config import PLATFORM, enable_verbose_mode, get_target_arch -from lib.util import execute_stdout, get_electron_version, safe_mkdir, \ - update_node_modules, update_electron_modules - - -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - - -def main(): - os.chdir(SOURCE_ROOT) - - args = parse_args() - config = args.configuration - - if args.verbose: - enable_verbose_mode() - - spec_modules = os.path.join(SOURCE_ROOT, 'spec', 'node_modules') - out_dir = os.path.join(SOURCE_ROOT, 'out', config) - version = get_electron_version() - node_dir = os.path.join(out_dir, 'node-{0}'.format(version)) - - # Create node headers - script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py') - execute_stdout([sys.executable, script_path, '--version', version, - '--directory', out_dir]) - - if PLATFORM == 'win32': - lib_dir = os.path.join(node_dir, 'Release') - safe_mkdir(lib_dir) - iojs_lib = os.path.join(lib_dir, 'iojs.lib') - atom_lib = os.path.join(out_dir, 'node.dll.lib') - shutil.copy2(atom_lib, iojs_lib) - node_lib = os.path.join(lib_dir, 'node.lib') - shutil.copy2(atom_lib, node_lib) - - # Native modules can only be compiled against release builds on Windows - if config[0] == 'R' or PLATFORM != 'win32': - update_electron_modules(os.path.dirname(spec_modules), get_target_arch(), - node_dir) - else: - update_node_modules(os.path.dirname(spec_modules)) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Rebuild native test modules') - parser.add_argument('-v', '--verbose', - action='store_true', - help='Prints the output of the subprocesses') - parser.add_argument('-c', '--configuration', - help='Build configuration to rebuild modules against', - default='D', - required=False) - return parser.parse_args() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/script/serve-node-headers.py b/script/serve-node-headers.py deleted file mode 100755 index 2d71a1a4881c..000000000000 --- a/script/serve-node-headers.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -import argparse -import atexit -import os -import shutil -import sys -import tarfile -import time - -from subprocess import Popen, PIPE -from lib.util import execute_stdout - -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') - - -def main(): - args = parse_args() - header_dir = os.path.join(DIST_DIR, args.version) - - # Generate Headers - script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py') - execute_stdout([sys.executable, script_path, '--version', args.version, - '--directory', header_dir]) - - # Launch server - script_path = os.path.join(SOURCE_ROOT, 'node_modules', 'serve', 'bin', - 'serve.js') - server = Popen(['node', script_path, '--port=' + args.port], stdout=PIPE, - cwd=DIST_DIR) - def cleanup(): - server.kill() - atexit.register(cleanup) - - time.sleep(1) - - # Generate Checksums - script_path = os.path.join(SOURCE_ROOT, 'script', 'upload-node-checksums.py') - execute_stdout([sys.executable, script_path, '--version', args.version, - '--dist-url', 'http://localhost:' + args.port, - '--target-dir', header_dir]) - - print("Point your npm config at 'http://localhost:" + args.port + "'") - server.wait() - -def parse_args(): - parser = argparse.ArgumentParser(description='create node header tarballs') - parser.add_argument('-v', '--version', help='Specify the version', - required=True) - parser.add_argument('-p', '--port', help='Specify port to run local server', - default='4321') - return parser.parse_args() - -if __name__ == '__main__': - sys.exit(main()) diff --git a/script/upload-index-json.py b/script/upload-index-json.py index 55d0675d0707..05c6f38466bd 100755 --- a/script/upload-index-json.py +++ b/script/upload-index-json.py @@ -5,10 +5,10 @@ import sys import urllib2 from lib.config import s3_config -from lib.util import s3put, scoped_cwd, safe_mkdir +from lib.util import s3put, scoped_cwd, safe_mkdir, get_out_dir SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'D') +OUT_DIR = get_out_dir() BASE_URL = 'https://electron-metadumper.herokuapp.com/?version=' diff --git a/script/upload-node-checksums.py b/script/upload-node-checksums.py index 3ed20a86fc8e..4d301ce2b937 100755 --- a/script/upload-node-checksums.py +++ b/script/upload-node-checksums.py @@ -10,7 +10,6 @@ import tempfile from lib.config import s3_config from lib.util import download, rm_rf, s3put, safe_mkdir - DIST_URL = 'https://atom.io/download/electron/' diff --git a/script/upload-node-headers.py b/script/upload-node-headers.py index 6c19d4f879c5..df8de4a65baf 100755 --- a/script/upload-node-headers.py +++ b/script/upload-node-headers.py @@ -7,12 +7,19 @@ import shutil import sys from lib.config import PLATFORM, get_target_arch, s3_config -from lib.util import safe_mkdir, scoped_cwd, s3put - +from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') +DIST_DIR = get_dist_dir() +OUT_DIR = get_out_dir() +GEN_DIR = os.path.join(OUT_DIR, 'gen') + +HEADER_TAR_NAMES = [ + 'node-{0}.tar.gz', + 'node-{0}-headers.tar.gz', + 'iojs-{0}.tar.gz', + 'iojs-{0}-headers.tar.gz' +] def main(): args = parse_args() @@ -30,10 +37,15 @@ def parse_args(): def upload_node(bucket, access_key, secret_key, version): - with scoped_cwd(DIST_DIR): - s3put(bucket, access_key, secret_key, DIST_DIR, + with scoped_cwd(GEN_DIR): + generated_tar = os.path.join(GEN_DIR, 'node_headers.tar.gz') + for header_tar in HEADER_TAR_NAMES: + versioned_header_tar = header_tar.format(version) + shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar)) + + s3put(bucket, access_key, secret_key, GEN_DIR, 'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz')) - s3put(bucket, access_key, secret_key, DIST_DIR, + s3put(bucket, access_key, secret_key, GEN_DIR, 'atom-shell/dist/{0}'.format(version), glob.glob('iojs-*.tar.gz')) if PLATFORM == 'win32': @@ -48,11 +60,11 @@ def upload_node(bucket, access_key, secret_key, version): safe_mkdir(os.path.dirname(node_lib)) safe_mkdir(os.path.dirname(iojs_lib)) - # Copy atom.lib to node.lib, iojs.lib and v4 node.lib - atom_lib = os.path.join(OUT_DIR, 'node.dll.lib') - shutil.copy2(atom_lib, node_lib) - shutil.copy2(atom_lib, iojs_lib) - shutil.copy2(atom_lib, v4_node_lib) + # Copy electron.lib to node.lib and iojs.lib. + electron_lib = os.path.join(OUT_DIR, 'electron.lib') + shutil.copy2(electron_lib, node_lib) + shutil.copy2(electron_lib, iojs_lib) + shutil.copy2(electron_lib, v4_node_lib) # Upload the node.lib. s3put(bucket, access_key, secret_key, DIST_DIR, diff --git a/script/upload-symbols.py b/script/upload-symbols.py index 34b7ab15a919..2e9b43558100 100755 --- a/script/upload-symbols.py +++ b/script/upload-symbols.py @@ -5,25 +5,24 @@ import glob import sys from lib.config import PLATFORM, s3_config, enable_verbose_mode -from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put - +from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put, \ + get_out_dir SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') -RELEASE_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') +RELEASE_DIR = get_out_dir() +GEN_DIR = os.path.join(RELEASE_DIR, 'gen') PROJECT_NAME = get_electron_branding()['project_name'] PRODUCT_NAME = get_electron_branding()['product_name'] if PLATFORM == 'win32': - SYMBOLS_DIR = os.path.join(DIST_DIR, 'symbols') + SYMBOLS_DIR = os.path.join(GEN_DIR, 'symbols') else: - SYMBOLS_DIR = os.path.join(DIST_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME)) + SYMBOLS_DIR = os.path.join(GEN_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME)) PDB_LIST = [ - os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME)), - os.path.join(RELEASE_DIR, 'node.dll.pdb') + os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME)) ] diff --git a/script/upload.py b/script/upload.py index 76dd6e6f1aab..394503de92f3 100755 --- a/script/upload.py +++ b/script/upload.py @@ -6,6 +6,7 @@ import errno import hashlib import json import os +import shutil import subprocess import sys import tempfile @@ -14,7 +15,8 @@ from io import StringIO from lib.config import PLATFORM, get_target_arch, get_env_var, s3_config, \ get_zip_name from lib.util import get_electron_branding, execute, get_electron_version, \ - parse_version, scoped_cwd, s3put + parse_version, scoped_cwd, s3put, get_electron_exec, \ + get_out_dir, GN_SRC_DIR ELECTRON_REPO = 'electron/electron' @@ -24,8 +26,7 @@ PROJECT_NAME = get_electron_branding()['project_name'] PRODUCT_NAME = get_electron_branding()['product_name'] SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') +OUT_DIR = get_out_dir() DIST_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION) SYMBOLS_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'symbols') @@ -58,40 +59,55 @@ def main(): 'You have to pass --overwrite to overwrite a published release' # Upload Electron files. - upload_electron(release, os.path.join(DIST_DIR, DIST_NAME), args) + # Rename dist.zip to get_zip_name('electron', version, suffix='') + electron_zip = os.path.join(OUT_DIR, DIST_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'dist.zip'), electron_zip) + upload_electron(release, electron_zip, args) if get_target_arch() != 'mips64el': - upload_electron(release, os.path.join(DIST_DIR, SYMBOLS_NAME), args) + symbols_zip = os.path.join(OUT_DIR, SYMBOLS_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'symobls.zip'), symbols_zip) + upload_electron(release, symbols_zip, args) if PLATFORM == 'darwin': - api_path = os.path.join(SOURCE_ROOT, 'electron-api.json') + api_path = os.path.join(OUT_DIR, 'electron-api.json') upload_electron(release, api_path, args) - ts_defs_path = os.path.join(SOURCE_ROOT, 'electron.d.ts') + ts_defs_path = os.path.join(OUT_DIR, 'electron.d.ts') upload_electron(release, ts_defs_path, args) - - upload_electron(release, os.path.join(DIST_DIR, DSYM_NAME), args) + dsym_zip = os.path.join(OUT_DIR, DSYM_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'dsym.zip'), dsym_zip) + upload_electron(release, dsym_zip, args) elif PLATFORM == 'win32': - upload_electron(release, os.path.join(DIST_DIR, PDB_NAME), args) + upload_electron(release, os.path.join(OUT_DIR, PDB_NAME), args) # Upload free version of ffmpeg. ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, ffmpeg), args) + ffmpeg_zip = os.path.join(OUT_DIR, ffmpeg) + shutil.copy2(os.path.join(OUT_DIR, 'ffmpeg.zip'), ffmpeg_zip) + upload_electron(release, ffmpeg_zip, args) chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, chromedriver), args) - mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) + chromedriver_zip = os.path.join(OUT_DIR, chromedriver) + shutil.copy2(os.path.join(OUT_DIR, 'chromedriver.zip'), chromedriver_zip) + upload_electron(release, chromedriver_zip, args) + mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) + mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot) if get_target_arch().startswith('arm'): + # Upload the native mksnapshot as mksnapshot.zip + shutil.copy2(os.path.join(GN_SRC_DIR, 'out', 'native_mksnapshot', + 'native_mksnapshot.zip'), mksnapshot_zip) + upload_electron(release, mksnapshot_zip, args) # Upload the x64 binary for arm/arm64 mksnapshot mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64') - upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) + mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot) + + shutil.copy2(os.path.join(OUT_DIR, 'mksnapshot.zip'), mksnapshot_zip) + upload_electron(release, mksnapshot_zip, args) if not tag_exists and not args.upload_to_s3: # Upload symbols to symbol server. run_python_script('upload-symbols.py') if PLATFORM == 'win32': - # Upload node headers. - run_python_script('create-node-headers.py', '-v', args.version) run_python_script('upload-node-headers.py', '-v', args.version) @@ -123,16 +139,7 @@ def get_electron_build_version(): if get_target_arch().startswith('arm') or os.environ.has_key('CI'): # In CI we just build as told. return ELECTRON_VERSION - if PLATFORM == 'darwin': - electron = os.path.join(SOURCE_ROOT, 'out', 'R', - '{0}.app'.format(PRODUCT_NAME), 'Contents', - 'MacOS', PRODUCT_NAME) - elif PLATFORM == 'win32': - electron = os.path.join(SOURCE_ROOT, 'out', 'R', - '{0}.exe'.format(PROJECT_NAME)) - else: - electron = os.path.join(SOURCE_ROOT, 'out', 'R', PROJECT_NAME) - + electron = get_electron_exec() return subprocess.check_output([electron, '--version']).strip() diff --git a/script/zip-symbols.py b/script/zip-symbols.py new file mode 100644 index 000000000000..7201fcf0629a --- /dev/null +++ b/script/zip-symbols.py @@ -0,0 +1,37 @@ +import glob +import os +import sys + +from lib.config import PLATFORM, get_target_arch +from lib.util import scoped_cwd, get_electron_version, make_zip, \ + electron_gyp, get_out_dir + +ELECTRON_VERSION = get_electron_version() +PROJECT_NAME = electron_gyp()['project_name%'] +OUT_DIR = get_out_dir() + +def main(): + if get_target_arch() == 'mips64el': + return + + dist_name = 'symbols.zip' + zip_file = os.path.join(OUT_DIR, dist_name) + licenses = ['LICENSE', 'LICENSES.chromium.html', 'version'] + + with scoped_cwd(OUT_DIR): + dirs = ['{0}.breakpad.syms'.format(PROJECT_NAME)] + make_zip(zip_file, licenses, dirs) + + if PLATFORM == 'darwin': + dsym_name = 'dsym.zip' + with scoped_cwd(OUT_DIR): + dsyms = glob.glob('*.dSYM') + make_zip(os.path.join(OUT_DIR, dsym_name), licenses, dsyms) + elif PLATFORM == 'win32': + pdb_name = 'pdb.zip' + with scoped_cwd(OUT_DIR): + pdbs = glob.glob('*.pdb') + make_zip(os.path.join(OUT_DIR, pdb_name), pdbs + licenses, []) + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/spec/api-notification-dbus-spec.js b/spec/api-notification-dbus-spec.js index beb3748b64f6..4046898367e2 100644 --- a/spec/api-notification-dbus-spec.js +++ b/spec/api-notification-dbus-spec.js @@ -2,7 +2,7 @@ // with the session bus. This requires python-dbusmock to be installed and // running at $DBUS_SESSION_BUS_ADDRESS. // -// script/test.py spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS. +// script/spec-runner.js spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS. // // See https://pypi.python.org/pypi/python-dbusmock to read about dbusmock. diff --git a/spec/api-power-monitor-spec.js b/spec/api-power-monitor-spec.js index 3558d9bcb438..60401d5bccaa 100644 --- a/spec/api-power-monitor-spec.js +++ b/spec/api-power-monitor-spec.js @@ -1,7 +1,7 @@ // For these tests we use a fake DBus daemon to verify powerMonitor module // interaction with the system bus. This requires python-dbusmock installed and // running (with the DBUS_SYSTEM_BUS_ADDRESS environment variable set). -// script/test.py will take care of spawning the fake DBus daemon and setting +// script/spec-runner.js will take care of spawning the fake DBus daemon and setting // DBUS_SYSTEM_BUS_ADDRESS when python-dbusmock is installed. // // See https://pypi.python.org/pypi/python-dbusmock for more information about diff --git a/vsts.yml b/vsts.yml index 69a988cdbf94..454a7299fb9e 100644 --- a/vsts.yml +++ b/vsts.yml @@ -61,7 +61,7 @@ jobs: - bash: | cd src gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" - ninja -C out/ffmpeg third_party/ffmpeg + ninja -C out/ffmpeg electron:electron_ffmpeg_zip displayName: Non proprietary ffmpeg build condition: and(succeeded(), eq(variables['RUN_TESTS'], '1')) @@ -70,6 +70,7 @@ jobs: # Build needed dump_syms executable ninja -C out/Default third_party/breakpad:dump_syms electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py displayName: Ninja build app condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) @@ -115,6 +116,18 @@ jobs: ninja -C out/Default electron:electron_mksnapshot_zip displayName: Build mksnapshot and zip + - bash: | + cd src/electron + if [ "$UPLOAD_TO_S3" != "1" ]; then + echo 'Uploading Electron release distribution to github releases' + ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py + else + echo 'Uploading Electron release distribution to s3' + ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py --upload_to_s3 + fi + name: Upload_distribution + condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) + - task: PublishTestResults@2 displayName: Publish Test Results inputs: