From 4b0f335aba7fe701644c4238a374faf67773cdc8 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 15:38:06 +1000 Subject: [PATCH 01/14] chore: initial work for running releases of GN build --- .circleci/config.yml | 70 ++++++++++++++++++++++++++++++++- appveyor.yml | 13 ++++++ script/ci-release-build.js | 40 +++++++++---------- script/dump-symbols.py | 2 +- script/lib/util.py | 8 ++++ script/upload-index-json.py | 2 + script/upload-node-checksums.py | 3 +- script/upload-node-headers.py | 3 +- script/upload-symbols.py | 3 +- script/upload.py | 2 + script/zip-symbols.py | 37 +++++++++++++++++ vsts.yml | 15 ++++++- 12 files changed, 172 insertions(+), 26 deletions(-) create mode 100644 script/zip-symbols.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ab4259d9ad1..8b295decec16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -179,6 +179,10 @@ step-persist-data-for-tests: &step-persist-data-for-tests # Build artifacts - src/out/Default/dist.zip - src/out/Default/gen/node_headers + - src/out/Default/chromedriver.zip + - src/out/Default/dsym.zip + - src/out/Default/symbols.zip + - src/out/Default/mksnapshot.zip step-electron-dist-unzip: &step-electron-dist-unzip run: @@ -201,7 +205,7 @@ step-ffmpeg-build: &step-ffmpeg-build name: Non proprietary ffmpeg build command: | cd src - ninja -C out/ffmpeg third_party/ffmpeg + ninja -C out/ffmpeg electron:electron_ffmpeg_zip step-verify-ffmpeg: &step-verify-ffmpeg run: @@ -245,6 +249,7 @@ step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols # Build needed dump_syms executable ninja -C out/Default third_party/breakpad:dump_syms electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py fi @@ -352,6 +357,10 @@ steps-native-mksnapshot-build: &steps-native-mksnapshot-build - store_artifacts: path: src/out/native_mksnapshot/native_mksnapshot.zip destination: native_mksnapshot.zip + - persist_to_workspace: + root: . + paths: + - src/out/Default/mksnapshot.zip steps-ffmpeg-build: &steps-ffmpeg-build steps: @@ -370,6 +379,7 @@ steps-ffmpeg-build: &steps-ffmpeg-build root: . paths: - src/out/ffmpeg/libffmpeg.so + - src/out/ffmpeg/ffmpeg.zip steps-native-tests: &steps-native-tests steps: @@ -776,6 +786,64 @@ workflows: requires: - linux-arm64-checkout + release-linux: + jobs: + - linux-checkout + - linux-arm-checkout + - linux-arm64-checkout + + - linux-x64-release: + requires: + - linux-checkout + - linux-x64-ffmpeg: + requires: + - linux-checkout + - linux-x64-publish: + requires: + - linux-x64-release + - linux-x64-ffmpeg + + - linux-ia32-release: + requires: + - linux-checkout + - linux-ia32-ffmpeg: + requires: + - linux-checkout + - linux-ia32-publish: + requires: + - linux-ia32-release + - linux-ia32-ffmpeg + + - linux-arm-release: + requires: + - linux-arm-checkout + - linux-arm-ffmpeg: + requires: + - linux-arm-checkout + - linux-arm-mksnapshot: + requires: + - linux-arm-checkout + - linux-arm-publish: + requires: + - linux-arm-release + - linux-arm-ffmpeg + - linux-arm-mksnapshot + + - linux-arm64-release: + requires: + - linux-arm64-checkout + - linux-arm64-ffmpeg: + requires: + - linux-arm64-checkout + - linux-arm64-mksnapshot: + requires: + - linux-arm64-checkout + - linux-arm64-publish: + requires: + - linux-arm64-release + - linux-arm64-ffmpeg + - linux-arm64-mksnapshot + build-mac-fork-prs: jobs: - osx-testing: diff --git a/appveyor.yml b/appveyor.yml index d0871e5cdec0..17dd2b499d1e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,6 +38,7 @@ build_script: - ps: >- if ($env:GN_CONFIG -eq 'release') { python electron\script\dump-symbols.py -d %cd%/out/Default/electron.breakpad.syms + python electron\script\zip-symbols.py appveyor PushArtifact out/Default/electron.breakpad.syms } test_script: @@ -52,3 +53,15 @@ test_script: } - cd electron - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci ) +deploy_script: + - cd electron + - ps: >- + if (Test-Path Env:\ELECTRON_RELEASE) { + if (Test-Path Env:\UPLOAD_TO_S3) { + Write-Output "Uploading Electron release distribution to s3" + & python script\upload.py --upload_to_s3 + } else { + Write-Output "Uploading Electron release distribution to github releases" + & python script\upload.py + } + } \ No newline at end of file diff --git a/script/ci-release-build.js b/script/ci-release-build.js index 04c8cb5a7dde..b78e7d97f563 100644 --- a/script/ci-release-build.js +++ b/script/ci-release-build.js @@ -6,16 +6,21 @@ const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds' const vstsURL = 'https://github.visualstudio.com/electron/_apis/build' const appVeyorJobs = { - 'electron-x64': 'electron', - 'electron-ia32': 'electron-39ng6' + 'electron-x64': 'electron-n7wrc', + 'electron-ia32': 'electron-egxcs' } +// TODO: Enable the Build Processing preview to get the ability to trigger workflows +// programatically. +// - https://circleci.com/docs/2.0/build-processing/ + +// TODO: Update sudowoodo to somehow support monitoring workflows and individual +// builds on legacy brances const circleCIJobs = [ - 'electron-linux-arm', - 'electron-linux-arm64', - 'electron-linux-ia32', - // 'electron-linux-mips64el', - 'electron-linux-x64' + 'linux-arm-release', + 'linux-arm64-release', + 'linux-ia32-release', + 'linux-x64-release' ] const vstsJobs = [ @@ -94,16 +99,13 @@ function buildAppVeyor (targetBranch, options) { async function callAppVeyor (targetBranch, job, options) { console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`) - let environmentVariables = {} - - if (options.ghRelease) { - environmentVariables.ELECTRON_RELEASE = 1 - } else { - environmentVariables.RUN_RELEASE_BUILD = 'true' + const environmentVariables = { + GN_CONFIG: 'release', + ELECTRON_RELEASE: 1 } - if (options.automaticRelease) { - environmentVariables.AUTO_RELEASE = 'true' + if (!options.ghRelease) { + environmentVariables.UPLOAD_TO_S3 = 1 } const requestOpts = { @@ -144,16 +146,14 @@ async function buildVSTS (targetBranch, options) { assert(vstsJobs.includes(options.job), `Unknown VSTS CI job name: ${options.job}. Valid values are: ${vstsJobs}.`) } console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`) - let environmentVariables = {} + const environmentVariables = { + ELECTRON_RELEASE: 1 + } if (!options.ghRelease) { environmentVariables.UPLOAD_TO_S3 = 1 } - if (options.automaticRelease) { - environmentVariables.AUTO_RELEASE = 'true' - } - let requestOpts = { url: `${vstsURL}/definitions?api-version=4.1`, auth: { diff --git a/script/dump-symbols.py b/script/dump-symbols.py index c4884e40b92e..7339692b387d 100755 --- a/script/dump-symbols.py +++ b/script/dump-symbols.py @@ -9,7 +9,7 @@ from lib.util import get_electron_branding, execute, rm_rf ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT)) -RELEASE_PATH = os.path.join('out', 'Release') +RELEASE_PATH = os.path.join('out', 'Default') def main(): args = parse_args() diff --git a/script/lib/util.py b/script/lib/util.py index d82bd6a11255..eccb09adbef5 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -21,6 +21,7 @@ import zipfile from lib.config import is_verbose_mode, PLATFORM from lib.env_util import get_vs_env +GN_SRC_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..')) BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'boto')) @@ -295,3 +296,10 @@ def make_version(major, minor, patch, pre = None): return major + '.' + minor + '.' + patch return major + "." + minor + "." + patch + '-' + pre +def get_out_dir(): + out_dir = 'Debug' + override = os.environ.get('ELECTRON_OUT_DIR') + if override is not None: + out_dir = override + return os.path.join(GN_SRC_DIR, 'out', out_dir) + diff --git a/script/upload-index-json.py b/script/upload-index-json.py index 55d0675d0707..658719b761ca 100755 --- a/script/upload-index-json.py +++ b/script/upload-index-json.py @@ -7,6 +7,8 @@ import urllib2 from lib.config import s3_config from lib.util import s3put, scoped_cwd, safe_mkdir +# TODO: Update this entire file to point at the correct file names in the out +# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'D') diff --git a/script/upload-node-checksums.py b/script/upload-node-checksums.py index 3ed20a86fc8e..20128b7026a1 100755 --- a/script/upload-node-checksums.py +++ b/script/upload-node-checksums.py @@ -10,7 +10,8 @@ import tempfile from lib.config import s3_config from lib.util import download, rm_rf, s3put, safe_mkdir - +# TODO: Update this entire file to point at the correct file names in the out +# directory DIST_URL = 'https://atom.io/download/electron/' diff --git a/script/upload-node-headers.py b/script/upload-node-headers.py index 6c19d4f879c5..904833c41f5a 100755 --- a/script/upload-node-headers.py +++ b/script/upload-node-headers.py @@ -9,7 +9,8 @@ import sys from lib.config import PLATFORM, get_target_arch, s3_config from lib.util import safe_mkdir, scoped_cwd, s3put - +# TODO: Update this entire file to point at the correct file names in the out +# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') diff --git a/script/upload-symbols.py b/script/upload-symbols.py index 34b7ab15a919..d3cfe0cdcbe7 100755 --- a/script/upload-symbols.py +++ b/script/upload-symbols.py @@ -7,7 +7,8 @@ import sys from lib.config import PLATFORM, s3_config, enable_verbose_mode from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put - +# TODO: Update this entire file to point at the correct file names in the out +# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') RELEASE_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') diff --git a/script/upload.py b/script/upload.py index 76dd6e6f1aab..fbea7812925c 100755 --- a/script/upload.py +++ b/script/upload.py @@ -33,6 +33,8 @@ DSYM_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'dsym') PDB_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'pdb') +# TODO: Update this entire file to point at the correct file names in the out +# directory def main(): args = parse_args() if args.upload_to_s3: diff --git a/script/zip-symbols.py b/script/zip-symbols.py new file mode 100644 index 000000000000..7201fcf0629a --- /dev/null +++ b/script/zip-symbols.py @@ -0,0 +1,37 @@ +import glob +import os +import sys + +from lib.config import PLATFORM, get_target_arch +from lib.util import scoped_cwd, get_electron_version, make_zip, \ + electron_gyp, get_out_dir + +ELECTRON_VERSION = get_electron_version() +PROJECT_NAME = electron_gyp()['project_name%'] +OUT_DIR = get_out_dir() + +def main(): + if get_target_arch() == 'mips64el': + return + + dist_name = 'symbols.zip' + zip_file = os.path.join(OUT_DIR, dist_name) + licenses = ['LICENSE', 'LICENSES.chromium.html', 'version'] + + with scoped_cwd(OUT_DIR): + dirs = ['{0}.breakpad.syms'.format(PROJECT_NAME)] + make_zip(zip_file, licenses, dirs) + + if PLATFORM == 'darwin': + dsym_name = 'dsym.zip' + with scoped_cwd(OUT_DIR): + dsyms = glob.glob('*.dSYM') + make_zip(os.path.join(OUT_DIR, dsym_name), licenses, dsyms) + elif PLATFORM == 'win32': + pdb_name = 'pdb.zip' + with scoped_cwd(OUT_DIR): + pdbs = glob.glob('*.pdb') + make_zip(os.path.join(OUT_DIR, pdb_name), pdbs + licenses, []) + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/vsts.yml b/vsts.yml index 69a988cdbf94..454a7299fb9e 100644 --- a/vsts.yml +++ b/vsts.yml @@ -61,7 +61,7 @@ jobs: - bash: | cd src gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" - ninja -C out/ffmpeg third_party/ffmpeg + ninja -C out/ffmpeg electron:electron_ffmpeg_zip displayName: Non proprietary ffmpeg build condition: and(succeeded(), eq(variables['RUN_TESTS'], '1')) @@ -70,6 +70,7 @@ jobs: # Build needed dump_syms executable ninja -C out/Default third_party/breakpad:dump_syms electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py displayName: Ninja build app condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) @@ -115,6 +116,18 @@ jobs: ninja -C out/Default electron:electron_mksnapshot_zip displayName: Build mksnapshot and zip + - bash: | + cd src/electron + if [ "$UPLOAD_TO_S3" != "1" ]; then + echo 'Uploading Electron release distribution to github releases' + ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py + else + echo 'Uploading Electron release distribution to s3' + ELECTRON_S3_BUCKET="$(s3_bucket)" ELECTRON_S3_ACCESS_KEY="$(s3_access_key)" ELECTRON_S3_SECRET_KEY="$(s3_secret_key)" ELECTRON_GITHUB_TOKEN="$(github_token)" script/upload.py --upload_to_s3 + fi + name: Upload_distribution + condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1')) + - task: PublishTestResults@2 displayName: Publish Test Results inputs: From 2adb44a402e9f9d9f98554d91180a70daea302a3 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 15:49:02 +1000 Subject: [PATCH 02/14] chore: remove unused files to simply out_dir replacements --- docs/development/upgrading-chromium.md | 2 +- script/dump-symbols.py | 4 +- script/rebuild-test-modules.py | 66 -------------------------- spec/api-notification-dbus-spec.js | 2 +- spec/api-power-monitor-spec.js | 2 +- 5 files changed, 5 insertions(+), 71 deletions(-) delete mode 100755 script/rebuild-test-modules.py diff --git a/docs/development/upgrading-chromium.md b/docs/development/upgrading-chromium.md index 0f6fb66d7f6a..15edfe0bdbbf 100644 --- a/docs/development/upgrading-chromium.md +++ b/docs/development/upgrading-chromium.md @@ -97,7 +97,7 @@ This is an overview of the steps needed to upgrade Chromium in Electron. - Add more build flags to disable features in build-time. When a Debug build of Electron succeeds, run the tests: -`$ ./script/test.py` +`$ npm run tst` Fix the failing tests. Follow all the steps above to fix Electron code on all supported platforms. diff --git a/script/dump-symbols.py b/script/dump-symbols.py index 7339692b387d..f8f4597d31fe 100755 --- a/script/dump-symbols.py +++ b/script/dump-symbols.py @@ -5,11 +5,11 @@ import os import sys from lib.config import PLATFORM, enable_verbose_mode, is_verbose_mode -from lib.util import get_electron_branding, execute, rm_rf +from lib.util import get_electron_branding, execute, rm_rf, get_out_dir ELECTRON_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) SOURCE_ROOT = os.path.abspath(os.path.dirname(ELECTRON_ROOT)) -RELEASE_PATH = os.path.join('out', 'Default') +RELEASE_PATH = get_out_dir() def main(): args = parse_args() diff --git a/script/rebuild-test-modules.py b/script/rebuild-test-modules.py deleted file mode 100755 index abf1800e33ce..000000000000 --- a/script/rebuild-test-modules.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -import argparse -import os -import shutil -import subprocess -import sys - -from lib.config import PLATFORM, enable_verbose_mode, get_target_arch -from lib.util import execute_stdout, get_electron_version, safe_mkdir, \ - update_node_modules, update_electron_modules - - -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - - -def main(): - os.chdir(SOURCE_ROOT) - - args = parse_args() - config = args.configuration - - if args.verbose: - enable_verbose_mode() - - spec_modules = os.path.join(SOURCE_ROOT, 'spec', 'node_modules') - out_dir = os.path.join(SOURCE_ROOT, 'out', config) - version = get_electron_version() - node_dir = os.path.join(out_dir, 'node-{0}'.format(version)) - - # Create node headers - script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py') - execute_stdout([sys.executable, script_path, '--version', version, - '--directory', out_dir]) - - if PLATFORM == 'win32': - lib_dir = os.path.join(node_dir, 'Release') - safe_mkdir(lib_dir) - iojs_lib = os.path.join(lib_dir, 'iojs.lib') - atom_lib = os.path.join(out_dir, 'node.dll.lib') - shutil.copy2(atom_lib, iojs_lib) - node_lib = os.path.join(lib_dir, 'node.lib') - shutil.copy2(atom_lib, node_lib) - - # Native modules can only be compiled against release builds on Windows - if config[0] == 'R' or PLATFORM != 'win32': - update_electron_modules(os.path.dirname(spec_modules), get_target_arch(), - node_dir) - else: - update_node_modules(os.path.dirname(spec_modules)) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Rebuild native test modules') - parser.add_argument('-v', '--verbose', - action='store_true', - help='Prints the output of the subprocesses') - parser.add_argument('-c', '--configuration', - help='Build configuration to rebuild modules against', - default='D', - required=False) - return parser.parse_args() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/spec/api-notification-dbus-spec.js b/spec/api-notification-dbus-spec.js index beb3748b64f6..4046898367e2 100644 --- a/spec/api-notification-dbus-spec.js +++ b/spec/api-notification-dbus-spec.js @@ -2,7 +2,7 @@ // with the session bus. This requires python-dbusmock to be installed and // running at $DBUS_SESSION_BUS_ADDRESS. // -// script/test.py spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS. +// script/spec-runner.js spawns dbusmock, which sets DBUS_SESSION_BUS_ADDRESS. // // See https://pypi.python.org/pypi/python-dbusmock to read about dbusmock. diff --git a/spec/api-power-monitor-spec.js b/spec/api-power-monitor-spec.js index 3558d9bcb438..60401d5bccaa 100644 --- a/spec/api-power-monitor-spec.js +++ b/spec/api-power-monitor-spec.js @@ -1,7 +1,7 @@ // For these tests we use a fake DBus daemon to verify powerMonitor module // interaction with the system bus. This requires python-dbusmock installed and // running (with the DBUS_SYSTEM_BUS_ADDRESS environment variable set). -// script/test.py will take care of spawning the fake DBus daemon and setting +// script/spec-runner.js will take care of spawning the fake DBus daemon and setting // DBUS_SYSTEM_BUS_ADDRESS when python-dbusmock is installed. // // See https://pypi.python.org/pypi/python-dbusmock for more information about From 504581e3081e63f13177f88f2aeee1368e8114f7 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 15:50:01 +1000 Subject: [PATCH 03/14] chore: fix out_dir usage in upload-index-json.py --- script/upload-index-json.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/upload-index-json.py b/script/upload-index-json.py index 658719b761ca..05c6f38466bd 100755 --- a/script/upload-index-json.py +++ b/script/upload-index-json.py @@ -5,12 +5,10 @@ import sys import urllib2 from lib.config import s3_config -from lib.util import s3put, scoped_cwd, safe_mkdir +from lib.util import s3put, scoped_cwd, safe_mkdir, get_out_dir -# TODO: Update this entire file to point at the correct file names in the out -# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'D') +OUT_DIR = get_out_dir() BASE_URL = 'https://electron-metadumper.herokuapp.com/?version=' From 363cf1dec258648230f30d017d71576a59d8936f Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 16:47:02 +1000 Subject: [PATCH 04/14] chore: fix out_dir usage in upload-node-checksums.py --- script/upload-node-checksums.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/upload-node-checksums.py b/script/upload-node-checksums.py index 20128b7026a1..4d301ce2b937 100755 --- a/script/upload-node-checksums.py +++ b/script/upload-node-checksums.py @@ -10,8 +10,6 @@ import tempfile from lib.config import s3_config from lib.util import download, rm_rf, s3put, safe_mkdir -# TODO: Update this entire file to point at the correct file names in the out -# directory DIST_URL = 'https://atom.io/download/electron/' From c1705f2789b076e20c1fbe69711226bab3ae60af Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 16:48:07 +1000 Subject: [PATCH 05/14] chore: fix out_dir usage in upload-node-headers.py --- script/lib/util.py | 2 ++ script/upload-node-headers.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/script/lib/util.py b/script/lib/util.py index eccb09adbef5..f4dc95c3985f 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -303,3 +303,5 @@ def get_out_dir(): out_dir = override return os.path.join(GN_SRC_DIR, 'out', out_dir) +def get_dist_dir(): + return os.path.join(get_out_dir(), 'gen', 'electron_dist') diff --git a/script/upload-node-headers.py b/script/upload-node-headers.py index 904833c41f5a..f4c4a26b2acc 100755 --- a/script/upload-node-headers.py +++ b/script/upload-node-headers.py @@ -7,13 +7,13 @@ import shutil import sys from lib.config import PLATFORM, get_target_arch, s3_config -from lib.util import safe_mkdir, scoped_cwd, s3put +from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir # TODO: Update this entire file to point at the correct file names in the out # directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') +DIST_DIR = get_dist_dir() +OUT_DIR = get_out_dir() def main(): args = parse_args() @@ -31,6 +31,7 @@ def parse_args(): def upload_node(bucket, access_key, secret_key, version): + safe_mkdir(DIST_DIR) with scoped_cwd(DIST_DIR): s3put(bucket, access_key, secret_key, DIST_DIR, 'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz')) @@ -49,11 +50,11 @@ def upload_node(bucket, access_key, secret_key, version): safe_mkdir(os.path.dirname(node_lib)) safe_mkdir(os.path.dirname(iojs_lib)) - # Copy atom.lib to node.lib, iojs.lib and v4 node.lib - atom_lib = os.path.join(OUT_DIR, 'node.dll.lib') - shutil.copy2(atom_lib, node_lib) - shutil.copy2(atom_lib, iojs_lib) - shutil.copy2(atom_lib, v4_node_lib) + # Copy electron.lib to node.lib and iojs.lib. + electron_lib = os.path.join(OUT_DIR, 'electron.lib') + shutil.copy2(electron_lib, node_lib) + shutil.copy2(electron_lib, iojs_lib) + shutil.copy2(electron_lib, v4_node_lib) # Upload the node.lib. s3put(bucket, access_key, secret_key, DIST_DIR, From 85464c5b4635e97c0beb5baf6d5f280638b130e8 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 17:17:42 +1000 Subject: [PATCH 06/14] chore: ensure boto is initialized so that s3put works --- DEPS | 27 +++++++++++++++++++++++++++ script/upload-node-headers.py | 20 ++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index 029e9db03943..19fcb02031e8 100644 --- a/DEPS +++ b/DEPS @@ -65,6 +65,33 @@ hooks = [ 'pattern': 'src/electron/package.json', 'name': 'electron_npm_deps' }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.system("git submodule update --init --recursive");', + ], + 'pattern': 'src/electron', + 'name': 'electron_submodules' + }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("boto"); os.system("python setup.py build");', + ], + 'pattern': 'src/electron', + 'name': 'setup_boto', + }, + { + 'action': [ + 'python', + '-c', + 'import os; os.chdir("src"); os.chdir("electron"); os.chdir("vendor"); os.chdir("requests"); os.system("python setup.py build");', + ], + 'pattern': 'src/electron', + 'name': 'setup_requests', + } ] recursedeps = [ diff --git a/script/upload-node-headers.py b/script/upload-node-headers.py index f4c4a26b2acc..a431e826aed7 100755 --- a/script/upload-node-headers.py +++ b/script/upload-node-headers.py @@ -14,6 +14,14 @@ from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) DIST_DIR = get_dist_dir() OUT_DIR = get_out_dir() +GEN_DIR = os.path.join(OUT_DIR, 'gen') + +HEADER_TAR_NAMES = [ + 'node-{0}.tar.gz', + 'node-{0}-headers.tar.gz', + 'iojs-{0}.tar.gz', + 'iojs-{0}-headers.tar.gz' +] def main(): args = parse_args() @@ -31,11 +39,15 @@ def parse_args(): def upload_node(bucket, access_key, secret_key, version): - safe_mkdir(DIST_DIR) - with scoped_cwd(DIST_DIR): - s3put(bucket, access_key, secret_key, DIST_DIR, + with scoped_cwd(GEN_DIR): + generated_tar = os.path.join(GEN_DIR, 'node_headers.tar.gz') + for header_tar in HEADER_TAR_NAMES: + versioned_header_tar = header_tar.format(version) + shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar)) + + s3put(bucket, access_key, secret_key, GEN_DIR, 'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz')) - s3put(bucket, access_key, secret_key, DIST_DIR, + s3put(bucket, access_key, secret_key, GEN_DIR, 'atom-shell/dist/{0}'.format(version), glob.glob('iojs-*.tar.gz')) if PLATFORM == 'win32': From d0691dffdfbc7470e4c3e1c9e13994d7940d4d69 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 17:30:35 +1000 Subject: [PATCH 07/14] chore: remove serve-node-headers.py, unused in new testing setup and points at old out_dir --- script/serve-node-headers.py | 56 ------------------------------------ 1 file changed, 56 deletions(-) delete mode 100755 script/serve-node-headers.py diff --git a/script/serve-node-headers.py b/script/serve-node-headers.py deleted file mode 100755 index 2d71a1a4881c..000000000000 --- a/script/serve-node-headers.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -import argparse -import atexit -import os -import shutil -import sys -import tarfile -import time - -from subprocess import Popen, PIPE -from lib.util import execute_stdout - -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') - - -def main(): - args = parse_args() - header_dir = os.path.join(DIST_DIR, args.version) - - # Generate Headers - script_path = os.path.join(SOURCE_ROOT, 'script', 'create-node-headers.py') - execute_stdout([sys.executable, script_path, '--version', args.version, - '--directory', header_dir]) - - # Launch server - script_path = os.path.join(SOURCE_ROOT, 'node_modules', 'serve', 'bin', - 'serve.js') - server = Popen(['node', script_path, '--port=' + args.port], stdout=PIPE, - cwd=DIST_DIR) - def cleanup(): - server.kill() - atexit.register(cleanup) - - time.sleep(1) - - # Generate Checksums - script_path = os.path.join(SOURCE_ROOT, 'script', 'upload-node-checksums.py') - execute_stdout([sys.executable, script_path, '--version', args.version, - '--dist-url', 'http://localhost:' + args.port, - '--target-dir', header_dir]) - - print("Point your npm config at 'http://localhost:" + args.port + "'") - server.wait() - -def parse_args(): - parser = argparse.ArgumentParser(description='create node header tarballs') - parser.add_argument('-v', '--version', help='Specify the version', - required=True) - parser.add_argument('-p', '--port', help='Specify port to run local server', - default='4321') - return parser.parse_args() - -if __name__ == '__main__': - sys.exit(main()) From cba749e305a5db0719d26859adfecfc12ac18b44 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 27 Sep 2018 18:06:02 +1000 Subject: [PATCH 08/14] chore: fix appveyor deploy working dir --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 17dd2b499d1e..9f3542a5a6f5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -53,6 +53,7 @@ test_script: } - cd electron - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci ) + - cd .. deploy_script: - cd electron - ps: >- From 3643ce5f853b406356c7fbea015a1f6f829b6997 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Thu, 27 Sep 2018 10:58:57 -0400 Subject: [PATCH 09/14] Add publish jobs for CircleCI releases --- .circleci/config.yml | 223 +++++++++++++++++++++---------------- script/ci-release-build.js | 19 +--- 2 files changed, 130 insertions(+), 112 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b295decec16..fa11c589592f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,9 @@ env-testing-build: &env-testing-build env-release-build: &env-release-build GN_CONFIG: //electron/build/args/release.gn NOTIFY_SLACK: true + +env-publish-build: &env-publish-build + GN_CONFIG: //electron/build/args/release.gn ELECTRON_RELEASE: 1 env-browsertests: &env-browsertests @@ -44,11 +47,13 @@ env-arm: &env-arm GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True' GN_EXTRA_ARGS: 'target_cpu = "arm"' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm + BUILD_NATIVE_MKSNAPSHOT: 1 env-arm64: &env-arm64 GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True' GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64 + BUILD_NATIVE_MKSNAPSHOT: 1 env-mas: &env-mas GN_EXTRA_ARGS: 'is_mas_build = true' @@ -110,9 +115,11 @@ step-setup-env-for-build: &step-setup-env-for-build # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - # https://github.com/mozilla/sccache - SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" - echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + if [ "$ELECTRON_RELEASE" != "1" ]; then + # https://github.com/mozilla/sccache + SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" + echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV + fi step-install-nodejs-on-mac: &step-install-nodejs-on-mac run: @@ -128,7 +135,11 @@ step-electron-gn-gen: &step-electron-gn-gen name: Electron GN gen command: | cd src - gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS" + else + gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-electron-build: &step-electron-build run: @@ -179,10 +190,6 @@ step-persist-data-for-tests: &step-persist-data-for-tests # Build artifacts - src/out/Default/dist.zip - src/out/Default/gen/node_headers - - src/out/Default/chromedriver.zip - - src/out/Default/dsym.zip - - src/out/Default/symbols.zip - - src/out/Default/mksnapshot.zip step-electron-dist-unzip: &step-electron-dist-unzip run: @@ -198,7 +205,11 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn")'" $GN_EXTRA_ARGS" + else + gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + fi step-ffmpeg-build: &step-ffmpeg-build run: @@ -214,6 +225,11 @@ step-verify-ffmpeg: &step-verify-ffmpeg cd src python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg +step-ffmpeg-store: &step-ffmpeg-store + - store_artifacts: + path: src/out/ffmpeg/ffmpeg.zip + destination: ffmpeg.zip + step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing run: name: Setup for headless testing @@ -240,18 +256,47 @@ step-mksnapshot-store: &step-mksnapshot-store path: src/out/Default/mksnapshot.zip destination: mksnapshot.zip -step-maybe-generate-breakpad_symbols: &step-maybe-generate-breakpad_symbols +step-generate-breakpad_symbols: &step-generate-breakpad_symbols run: name: Generate breakpad symbols command: | - if [ "$ELECTRON_RELEASE" == "1" ]; then + cd src + # Build needed dump_syms executable + ninja -C out/Default third_party/breakpad:dump_syms + electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" + electron/script/zip-symbols.py + +step-native-mksnapshot-gn-gen: &step-native-mksnapshot-gn-gen + run: + name: native mksnapshot GN gen (if needed) + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then cd src - # Build needed dump_syms executable - ninja -C out/Default third_party/breakpad:dump_syms - electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms" - electron/script/zip-symbols.py + if [ "$ELECTRON_RELEASE" == "1" ]; then + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + else + gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" + fi + else + echo 'skipping native mksnapshot GN gen for non arm build' fi +step-native-mksnapshot-build: &step-native-mksnapshot-build + run: + name: native mksnapshot (arm/arm64) build + command: | + if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then + cd src + ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip + cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip + else + echo 'skipping native mksnapshot build for non arm build' + fi + +step-native-mksnapshot-store: &step-native-mksnapshot-store + store_artifacts: + path: src/out/native_mksnapshot/native_mksnapshot.zip + destination: native_mksnapshot.zip # Lists of steps. steps-checkout: &steps-checkout @@ -315,7 +360,6 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-electron-build - *step-electron-dist-build - *step-electron-dist-store - - *step-maybe-generate-breakpad_symbols # mksnapshot - *step-mksnapshot-build @@ -336,31 +380,47 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests - *step-maybe-notify-slack-failure - *step-maybe-notify-slack-success +steps-electron-build-for-publish: &steps-electron-build-for-publish + steps: + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-gclient-sync + - *step-setup-env-for-build + + # Electron app + - *step-electron-gn-gen + - *step-electron-build + - *step-electron-dist-build + - *step-electron-dist-store + - *step-generate-breakpad_symbols + + # mksnapshot + - *step-mksnapshot-build + - *step-mksnapshot-store + + # native_mksnapshot + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store + + # chromedriver + - *step-electron-chromedriver-build + - *step-electron-chromedriver-store + + # Node.js headers + - *step-nodejs-headers-build + + steps-native-mksnapshot-build: &steps-native-mksnapshot-build steps: - attach_workspace: at: . - *step-depot-tools-add-to-path - *step-setup-env-for-build - - - run: - name: native mksnapshot GN gen - command: | - cd src - gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS" - - run: - name: native mksnapshot (arm/arm64) build - command: | - cd src - ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip - cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip - - store_artifacts: - path: src/out/native_mksnapshot/native_mksnapshot.zip - destination: native_mksnapshot.zip - - persist_to_workspace: - root: . - paths: - - src/out/Default/mksnapshot.zip + - *step-native-mksnapshot-gn-gen + - *step-native-mksnapshot-build + - *step-native-mksnapshot-store steps-ffmpeg-build: &steps-ffmpeg-build steps: @@ -371,10 +431,8 @@ steps-ffmpeg-build: &steps-ffmpeg-build - *step-ffmpeg-gn-gen - *step-ffmpeg-build + - *step-ffmpeg-store - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.so - destination: libffmpeg.so - persist_to_workspace: root: . paths: @@ -486,9 +544,7 @@ steps-build-mac: &steps-build-mac # ffmpeg - *step-ffmpeg-gn-gen - *step-ffmpeg-build - - store_artifacts: - path: src/out/ffmpeg/libffmpeg.dylib - destination: libffmpeg.dylib + - *step-ffmpeg-store # It would be better to verify ffmpeg as a part of a test job, # but it requires `gn` to run, and it's complicated @@ -557,6 +613,12 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-x64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-ia32-debug: <<: *machine-linux-2xlarge environment: @@ -584,6 +646,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-ia32-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-ia32 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-debug: <<: *machine-linux-2xlarge environment: @@ -611,6 +680,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm-mksnapshot: <<: *machine-linux-medium environment: @@ -644,6 +720,13 @@ jobs: <<: *env-release-build <<: *steps-electron-build-for-tests + linux-arm64-publish: + <<: *machine-linux-2xlarge + environment: + <<: *env-arm64 + <<: *env-publish-build + <<: *steps-electron-build-for-publish + linux-arm64-mksnapshot: <<: *machine-linux-medium environment: @@ -786,64 +869,6 @@ workflows: requires: - linux-arm64-checkout - release-linux: - jobs: - - linux-checkout - - linux-arm-checkout - - linux-arm64-checkout - - - linux-x64-release: - requires: - - linux-checkout - - linux-x64-ffmpeg: - requires: - - linux-checkout - - linux-x64-publish: - requires: - - linux-x64-release - - linux-x64-ffmpeg - - - linux-ia32-release: - requires: - - linux-checkout - - linux-ia32-ffmpeg: - requires: - - linux-checkout - - linux-ia32-publish: - requires: - - linux-ia32-release - - linux-ia32-ffmpeg - - - linux-arm-release: - requires: - - linux-arm-checkout - - linux-arm-ffmpeg: - requires: - - linux-arm-checkout - - linux-arm-mksnapshot: - requires: - - linux-arm-checkout - - linux-arm-publish: - requires: - - linux-arm-release - - linux-arm-ffmpeg - - linux-arm-mksnapshot - - - linux-arm64-release: - requires: - - linux-arm64-checkout - - linux-arm64-ffmpeg: - requires: - - linux-arm64-checkout - - linux-arm64-mksnapshot: - requires: - - linux-arm64-checkout - - linux-arm64-publish: - requires: - - linux-arm64-release - - linux-arm64-ffmpeg - - linux-arm64-mksnapshot - build-mac-fork-prs: jobs: - osx-testing: diff --git a/script/ci-release-build.js b/script/ci-release-build.js index b78e7d97f563..42d88a9583c2 100644 --- a/script/ci-release-build.js +++ b/script/ci-release-build.js @@ -6,21 +6,15 @@ const buildAppVeyorURL = 'https://windows-ci.electronjs.org/api/builds' const vstsURL = 'https://github.visualstudio.com/electron/_apis/build' const appVeyorJobs = { - 'electron-x64': 'electron-n7wrc', - 'electron-ia32': 'electron-egxcs' + 'electron-x64': 'electron', + 'electron-ia32': 'electron-39ng6' } -// TODO: Enable the Build Processing preview to get the ability to trigger workflows -// programatically. -// - https://circleci.com/docs/2.0/build-processing/ - -// TODO: Update sudowoodo to somehow support monitoring workflows and individual -// builds on legacy brances const circleCIJobs = [ - 'linux-arm-release', - 'linux-arm64-release', - 'linux-ia32-release', - 'linux-x64-release' + 'linux-arm-publish', + 'linux-arm64-publish', + 'linux-ia32-publish', + 'linux-x64-publish' ] const vstsJobs = [ @@ -100,7 +94,6 @@ function buildAppVeyor (targetBranch, options) { async function callAppVeyor (targetBranch, job, options) { console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`) const environmentVariables = { - GN_CONFIG: 'release', ELECTRON_RELEASE: 1 } From 1c728876e64caa3f61b6791290fa0bcba1a4b670 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Thu, 27 Sep 2018 11:05:11 -0400 Subject: [PATCH 10/14] Fix yaml issue --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa11c589592f..701f6b169ed2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -226,9 +226,9 @@ step-verify-ffmpeg: &step-verify-ffmpeg python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg step-ffmpeg-store: &step-ffmpeg-store - - store_artifacts: - path: src/out/ffmpeg/ffmpeg.zip - destination: ffmpeg.zip + store_artifacts: + path: src/out/ffmpeg/ffmpeg.zip + destination: ffmpeg.zip step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing run: From e66a5f79a2003aca57d2b2678834d7a9c3d3b4dc Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Thu, 27 Sep 2018 13:14:13 -0700 Subject: [PATCH 11/14] updating directories and related functions --- script/lib/util.py | 8 +++++++ script/upload.py | 52 +++++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/script/lib/util.py b/script/lib/util.py index f4dc95c3985f..a40f94b5b43e 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -305,3 +305,11 @@ def get_out_dir(): def get_dist_dir(): return os.path.join(get_out_dir(), 'gen', 'electron_dist') + +def get_electron_exec(): + if sys.platform == 'darwin': + return 'out/{0}/Electron.app/Contents/MacOS/Electron'.format(get_out_dir()) + elif sys.platform == 'win32': + return 'out/{0}/electron.exe'.format(get_out_dir()) + elif sys.platform == 'linux': + return 'out/{0}/electron'.format(get_out_dir()) \ No newline at end of file diff --git a/script/upload.py b/script/upload.py index fbea7812925c..ad4d650d8cac 100755 --- a/script/upload.py +++ b/script/upload.py @@ -6,6 +6,7 @@ import errno import hashlib import json import os +import shutil import subprocess import sys import tempfile @@ -14,7 +15,8 @@ from io import StringIO from lib.config import PLATFORM, get_target_arch, get_env_var, s3_config, \ get_zip_name from lib.util import get_electron_branding, execute, get_electron_version, \ - parse_version, scoped_cwd, s3put + parse_version, scoped_cwd, s3put, get_electron_exec, \ + get_out_dir ELECTRON_REPO = 'electron/electron' @@ -24,8 +26,7 @@ PROJECT_NAME = get_electron_branding()['project_name'] PRODUCT_NAME = get_electron_branding()['product_name'] SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -OUT_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') +OUT_DIR = get_out_dir() DIST_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION) SYMBOLS_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'symbols') @@ -60,40 +61,48 @@ def main(): 'You have to pass --overwrite to overwrite a published release' # Upload Electron files. - upload_electron(release, os.path.join(DIST_DIR, DIST_NAME), args) + # Rename dist.zip to get_zip_name('electron', version, suffix='') + electron_zip = os.path.join(OUT_DIR, DIST_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'dist.zip'), electron_zip) + upload_electron(release, electron_zip, args) if get_target_arch() != 'mips64el': - upload_electron(release, os.path.join(DIST_DIR, SYMBOLS_NAME), args) + symbols_zip = os.path.join(OUT_DIR, SYMBOLS_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'symobls.zip'), symbols_zip) + upload_electron(release, symbols_zip, args) if PLATFORM == 'darwin': - api_path = os.path.join(SOURCE_ROOT, 'electron-api.json') + api_path = os.path.join(OUT_DIR, 'electron-api.json') upload_electron(release, api_path, args) - ts_defs_path = os.path.join(SOURCE_ROOT, 'electron.d.ts') + ts_defs_path = os.path.join(OUT_DIR, 'electron.d.ts') upload_electron(release, ts_defs_path, args) - - upload_electron(release, os.path.join(DIST_DIR, DSYM_NAME), args) + dsym_zip = os.path.join(OUT_DIR, DSYM_NAME) + shutil.copy2(os.path.join(OUT_DIR, 'dsym.zip'), dsym_zip) + upload_electron(release, dsym_zip, args) elif PLATFORM == 'win32': - upload_electron(release, os.path.join(DIST_DIR, PDB_NAME), args) + upload_electron(release, os.path.join(OUT_DIR, PDB_NAME), args) # Upload free version of ffmpeg. ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, ffmpeg), args) + ffmpeg_zip = os.path.join(OUT_DIR, ffmpeg) + shutil.copy2(os.path.join(OUT_DIR, 'ffmpeg.zip'), ffmpeg_zip) + upload_electron(release, ffmpeg_zip, args) chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, chromedriver), args) + chromedriver_zip = os.path.join(OUT_DIR, chromedriver) + shutil.copy2(os.path.join(OUT_DIR, 'chromedriver.zip'), chromedriver_zip) + upload_electron(release, chromedriver_zip, args) mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) - upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) + upload_electron(release, os.path.join(OUT_DIR, mksnapshot), args) if get_target_arch().startswith('arm'): # Upload the x64 binary for arm/arm64 mksnapshot mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64') - upload_electron(release, os.path.join(DIST_DIR, mksnapshot), args) + upload_electron(release, os.path.join(OUT_DIR, mksnapshot), args) if not tag_exists and not args.upload_to_s3: # Upload symbols to symbol server. run_python_script('upload-symbols.py') if PLATFORM == 'win32': - # Upload node headers. - run_python_script('create-node-headers.py', '-v', args.version) run_python_script('upload-node-headers.py', '-v', args.version) @@ -125,16 +134,7 @@ def get_electron_build_version(): if get_target_arch().startswith('arm') or os.environ.has_key('CI'): # In CI we just build as told. return ELECTRON_VERSION - if PLATFORM == 'darwin': - electron = os.path.join(SOURCE_ROOT, 'out', 'R', - '{0}.app'.format(PRODUCT_NAME), 'Contents', - 'MacOS', PRODUCT_NAME) - elif PLATFORM == 'win32': - electron = os.path.join(SOURCE_ROOT, 'out', 'R', - '{0}.exe'.format(PROJECT_NAME)) - else: - electron = os.path.join(SOURCE_ROOT, 'out', 'R', PROJECT_NAME) - + electron = get_electron_exec() return subprocess.check_output([electron, '--version']).strip() From 8df703a13b5dbfc4e02de2026cd27bc16a3d0d1a Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Thu, 27 Sep 2018 17:26:49 -0400 Subject: [PATCH 12/14] Update mksnapshot uploads --- script/upload.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/script/upload.py b/script/upload.py index ad4d650d8cac..8fc7aea01c56 100755 --- a/script/upload.py +++ b/script/upload.py @@ -16,7 +16,7 @@ from lib.config import PLATFORM, get_target_arch, get_env_var, s3_config, \ get_zip_name from lib.util import get_electron_branding, execute, get_electron_version, \ parse_version, scoped_cwd, s3put, get_electron_exec, \ - get_out_dir + get_out_dir, GN_SRC_DIR ELECTRON_REPO = 'electron/electron' @@ -91,13 +91,20 @@ def main(): chromedriver_zip = os.path.join(OUT_DIR, chromedriver) shutil.copy2(os.path.join(OUT_DIR, 'chromedriver.zip'), chromedriver_zip) upload_electron(release, chromedriver_zip, args) - mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) - upload_electron(release, os.path.join(OUT_DIR, mksnapshot), args) + mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION) + mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot) if get_target_arch().startswith('arm'): + # Upload the native mksnapshot as mksnapshot.zip + shutil.copy2(os.path.join(GN_SRC_DIR, 'out', 'native_mksnapshot', + 'native_mksnapshot.zip'), mksnapshot_zip) + upload_electron(release, mksnapshot_zip, args) # Upload the x64 binary for arm/arm64 mksnapshot mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64') - upload_electron(release, os.path.join(OUT_DIR, mksnapshot), args) + mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot) + + shutil.copy2(os.path.join(OUT_DIR, 'mksnapshot.zip'), mksnapshot_zip) + upload_electron(release, mksnapshot_zip, args) if not tag_exists and not args.upload_to_s3: # Upload symbols to symbol server. From 55765424f949eef7ac0c3763e6b630999eef85b3 Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 28 Sep 2018 11:19:00 +1000 Subject: [PATCH 13/14] chore: fix test command in docs --- docs/development/upgrading-chromium.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/upgrading-chromium.md b/docs/development/upgrading-chromium.md index 15edfe0bdbbf..bf75a36701cf 100644 --- a/docs/development/upgrading-chromium.md +++ b/docs/development/upgrading-chromium.md @@ -97,7 +97,7 @@ This is an overview of the steps needed to upgrade Chromium in Electron. - Add more build flags to disable features in build-time. When a Debug build of Electron succeeds, run the tests: -`$ npm run tst` +`$ npm run test` Fix the failing tests. Follow all the steps above to fix Electron code on all supported platforms. From 37f405859f860738c4ef80700913c83a5beecb4c Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 28 Sep 2018 11:24:25 +1000 Subject: [PATCH 14/14] chore: fix out_dir usage in upload-symbols.py --- script/lib/util.py | 2 ++ script/upload-node-headers.py | 2 -- script/upload-symbols.py | 16 +++++++--------- script/upload.py | 2 -- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/script/lib/util.py b/script/lib/util.py index a40f94b5b43e..26593413976b 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -303,6 +303,8 @@ def get_out_dir(): out_dir = override return os.path.join(GN_SRC_DIR, 'out', out_dir) +# NOTE: This path is not created by gn, it is used as a scratch zone by our +# upload scripts def get_dist_dir(): return os.path.join(get_out_dir(), 'gen', 'electron_dist') diff --git a/script/upload-node-headers.py b/script/upload-node-headers.py index a431e826aed7..df8de4a65baf 100755 --- a/script/upload-node-headers.py +++ b/script/upload-node-headers.py @@ -9,8 +9,6 @@ import sys from lib.config import PLATFORM, get_target_arch, s3_config from lib.util import safe_mkdir, scoped_cwd, s3put, get_out_dir, get_dist_dir -# TODO: Update this entire file to point at the correct file names in the out -# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) DIST_DIR = get_dist_dir() OUT_DIR = get_out_dir() diff --git a/script/upload-symbols.py b/script/upload-symbols.py index d3cfe0cdcbe7..2e9b43558100 100755 --- a/script/upload-symbols.py +++ b/script/upload-symbols.py @@ -5,26 +5,24 @@ import glob import sys from lib.config import PLATFORM, s3_config, enable_verbose_mode -from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put +from lib.util import get_electron_branding, execute, rm_rf, safe_mkdir, s3put, \ + get_out_dir -# TODO: Update this entire file to point at the correct file names in the out -# directory SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -DIST_DIR = os.path.join(SOURCE_ROOT, 'dist') -RELEASE_DIR = os.path.join(SOURCE_ROOT, 'out', 'R') +RELEASE_DIR = get_out_dir() +GEN_DIR = os.path.join(RELEASE_DIR, 'gen') PROJECT_NAME = get_electron_branding()['project_name'] PRODUCT_NAME = get_electron_branding()['product_name'] if PLATFORM == 'win32': - SYMBOLS_DIR = os.path.join(DIST_DIR, 'symbols') + SYMBOLS_DIR = os.path.join(GEN_DIR, 'symbols') else: - SYMBOLS_DIR = os.path.join(DIST_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME)) + SYMBOLS_DIR = os.path.join(GEN_DIR, '{0}.breakpad.syms'.format(PROJECT_NAME)) PDB_LIST = [ - os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME)), - os.path.join(RELEASE_DIR, 'node.dll.pdb') + os.path.join(RELEASE_DIR, '{0}.exe.pdb'.format(PROJECT_NAME)) ] diff --git a/script/upload.py b/script/upload.py index 8fc7aea01c56..394503de92f3 100755 --- a/script/upload.py +++ b/script/upload.py @@ -34,8 +34,6 @@ DSYM_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'dsym') PDB_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'pdb') -# TODO: Update this entire file to point at the correct file names in the out -# directory def main(): args = parse_args() if args.upload_to_s3: