From 5df6c5ecced7f991e09859ae5f562630c9c267ea Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Fri, 21 Aug 2020 20:30:04 -0700 Subject: [PATCH] build: more space on arm64 builds, running out of ideas here (#25043) * build: more space on arm64 builds, running out of ideas here * build: well this is kinda nuts but off we go * chore: rollback thingy * chore: build snapshot for realz * chore: do not delete hunspell * build: use the new magic extra-disk-space circle image * build: remove existing file, it is not a tree --- .circleci/config.yml | 97 ++++++++++++++++++--- BUILD.gn | 22 ++++- build/fake_v8_context_snapshot_generator.py | 8 ++ 3 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 build/fake_v8_context_snapshot_generator.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e7e5e2b51f..a89781c0720 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ machine-mac-large: &machine-mac-large machine-mac-large-arm: &machine-mac-large-arm resource_class: large macos: - xcode: "12.0.0-UA" + xcode: "12.0.0-large" # Build configurations options. env-testing-build: &env-testing-build @@ -137,8 +137,9 @@ env-arm: &env-arm TARGET_ARCH: arm env-apple-silicon: &env-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64"' + GN_EXTRA_ARGS: 'target_cpu = "arm64" use_prebuilt_v8_context_snapshot = true' TARGET_ARCH: arm64 + USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 env-arm64: &env-arm64 GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false' @@ -151,9 +152,10 @@ env-mas: &env-mas MAS_BUILD: 'true' env-mas-apple-silicon: &env-mas-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true' + GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true use_prebuilt_v8_context_snapshot = true' MAS_BUILD: 'true' TARGET_ARCH: arm64 + USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 # Misc build configuration options. env-enable-sccache: &env-enable-sccache @@ -344,23 +346,69 @@ step-get-more-space-on-mac: &step-get-more-space-on-mac sudo mv "$1" $TMPDIR/del-target/$(echo $1|shasum -a 256|head -n1|cut -d " " -f1) fi } + + strip_arm_deep() { + opwd=$(pwd) + cd $1 + f=$(find . -perm +111 -type f) + for fp in $f + do + if [[ $(file "$fp") == *"universal binary"* ]]; then + if [[ $(file "$fp") == *"arm64e)"* ]]; then + sudo lipo -remove arm64e "$fp" -o "$fp" || true + fi + if [[ $(file "$fp") == *"arm64)"* ]]; then + sudo lipo -remove arm64 "$fp" -o "$fp" || true + fi + fi + done + + cd $opwd + } + tmpify /Library/Developer/CoreSimulator + tmpify ~/Library/Developer/CoreSimulator tmpify $(xcode-select -p)/Platforms/AppleTVOS.platform tmpify $(xcode-select -p)/Platforms/iPhoneOS.platform tmpify $(xcode-select -p)/Platforms/WatchOS.platform tmpify $(xcode-select -p)/Platforms/WatchSimulator.platform tmpify $(xcode-select -p)/Platforms/AppleTVSimulator.platform tmpify $(xcode-select -p)/Platforms/iPhoneSimulator.platform + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/metal/ios + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift + tmpify $(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.0 tmpify ~/.rubies tmpify ~/Library/Caches/Homebrew tmpify /usr/local/Homebrew - if [ "$TARGET_ARCH" == "arm64" ]; then - tmpify "/System/Library/Desktop Pictures" - tmpify /System/Library/Templates/Data - tmpify /System/Library/Speech/Voices - tmpify "/System/Library/Screen Savers" - fi sudo rm -rf $TMPDIR/del-target + + if [ "$TARGET_ARCH" == "arm64" ]; then + sudo rm -rf "/System/Library/Desktop Pictures" + sudo rm -rf /System/Library/Templates/Data + sudo rm -rf /System/Library/Speech/Voices + sudo rm -rf "/System/Library/Screen Savers" + sudo rm -rf /System/Volumes/Data/Library/Developer/CommandLineTools/SDKs + sudo rm -rf "/System/Volumes/Data/Library/Application Support/Apple/Photos/Print Products" + sudo rm -rf /System/Volumes/Data/Library/Java + sudo rm -rf /System/Volumes/Data/Library/Ruby + sudo rm -rf /System/Volumes/Data/Library/Printers + sudo rm -rf /System/iOSSupport + sudo rm -rf /System/Applications/*.app + sudo rm -rf /System/Applications/Utilities/*.app + sudo rm -rf /System/Library/LinguisticData + sudo rm -rf /System/Volumes/Data/private/var/db/dyld/* + # sudo rm -rf /System/Library/Fonts/* + # sudo rm -rf /System/Library/PreferencePanes + sudo rm -rf /System/Library/AssetsV2/* + sudo rm -rf /Applications/Safari.app + sudo rm -rf ~/project/src/build/linux + sudo rm -rf ~/project/src/third_party/catapult/tracing/test_data + sudo rm -rf ~/project/src/third_party/angle/third_party/VK-GL-CTS + + # lipo off some huge binaries arm64 versions to save space + strip_arm_deep $(xcode-select -p)/../SharedFrameworks + strip_arm_deep /System/Volumes/Data/Library/Developer/CommandLineTools/usr + fi fi background: true @@ -460,6 +508,28 @@ step-electron-build: &step-electron-build rm -f src/out/Default/libffmpeg.so fi cd src + # Enable if things get really bad + # if [ "$TARGET_ARCH" == "arm64" ] &&[ "`uname`" == "Darwin" ]; then + # diskutil erasevolume HFS+ "xcode_disk" `hdiutil attach -nomount ram://12582912` + # mv /Applications/Xcode-12.beta.5.app /Volumes/xcode_disk/ + # ln -s /Volumes/xcode_disk/Xcode-12.beta.5.app /Applications/Xcode-12.beta.5.app + # fi + + # Lets generate a snapshot and mksnapshot and then delete all the x-compiled generated files to save space + if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" == "1" ]; then + ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + ninja -C out/Default tools/v8_context_snapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + touch out/Default/.electron_mksnapshot_zip_done + rm -rf out/Default/clang_x64_v8_arm64/obj + + # Regenerate because we just deleted some ninja files + if [ "$USE_GOMA" == "true" ]; then + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + else + gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + fi + fi ninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES node electron/script/check-symlinks.js @@ -709,8 +779,10 @@ step-mksnapshot-build: &step-mksnapshot-build name: mksnapshot build command: | cd src - ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES - gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + if [ ! -s "src/out/Default/.electron_mksnapshot_zip_done" ]; then + ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + fi if [ "`uname`" != "Darwin" ]; then if [ "$TARGET_ARCH" == "arm" ]; then electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot @@ -723,7 +795,7 @@ step-mksnapshot-build: &step-mksnapshot-build electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator fi fi - if [ "$SKIP_DIST_ZIP" != "1" ]; then + if [ ! -s "src/out/Default/.electron_mksnapshot_zip_done" ] && [ "$SKIP_DIST_ZIP" != "1" ]; then ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) fi @@ -942,6 +1014,7 @@ step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-c rm -rf src/third_party/WebKit/LayoutTests rm -rf third_party/electron_node/deps/openssl rm -rf third_party/electron_node/deps/v8 + rm -rf chrome/test/data/xr/webvr_info # Save the src cache based on the deps hash step-save-src-cache: &step-save-src-cache diff --git a/BUILD.gn b/BUILD.gn index 5dbde354078..6b538e1432b 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -49,6 +49,10 @@ if (is_linux) { } } +declare_args() { + use_prebuilt_v8_context_snapshot = false +} + branding = read_file("shell/app/BRANDING.json", "json") electron_project_name = branding.project_name electron_product_name = branding.product_name @@ -688,6 +692,15 @@ if (is_mac) { sources = [ "shell/common/resources/mac/MainMenu.xib" ] } + action("fake_v8_context_snapshot_generator") { + script = "build/fake_v8_context_snapshot_generator.py" + args = [ + rebase_path("$root_out_dir/v8_context_snapshot.bin"), + rebase_path("$root_out_dir/fake/v8_context_snapshot.bin"), + ] + outputs = [ "$root_out_dir/fake/v8_context_snapshot.bin" ] + } + bundle_data("electron_framework_resources") { public_deps = [ ":packed_resources" ] sources = [] @@ -698,8 +711,13 @@ if (is_mac) { if (v8_use_external_startup_data) { public_deps += [ "//v8" ] if (use_v8_context_snapshot) { - sources += [ "$root_out_dir/v8_context_snapshot.bin" ] - public_deps += [ "//tools/v8_context_snapshot" ] + if (use_prebuilt_v8_context_snapshot) { + sources += [ "$root_out_dir/fake/v8_context_snapshot.bin" ] + public_deps += [ ":fake_v8_context_snapshot_generator" ] + } else { + sources += [ "$root_out_dir/v8_context_snapshot.bin" ] + public_deps += [ "//tools/v8_context_snapshot" ] + } } else { sources += [ "$root_out_dir/snapshot_blob.bin" ] } diff --git a/build/fake_v8_context_snapshot_generator.py b/build/fake_v8_context_snapshot_generator.py new file mode 100644 index 00000000000..2309b22484b --- /dev/null +++ b/build/fake_v8_context_snapshot_generator.py @@ -0,0 +1,8 @@ +import os +import shutil +import sys + +if os.path.exists(sys.argv[2]): + os.remove(sys.argv[2]) + +shutil.copy(sys.argv[1], sys.argv[2])