build: add infra for reclient support (#40850)
* chore: add patch to always set macos platform for x-build * build: add infra for reclient support * build: override reclient version * build: use RBE in CI * chore: hardcode reclient fix version * build: lower process count on macOS * build: use large macOS instance for testing-arm64 * Revert "build: use large macOS instance for testing-arm64" This reverts commit 6844adfd00a5230e68234112dfd84caa50d3f621. * build: login in via helper not writing file * chore: update patches * build: use recelint from DEPS * build: fix windows reproxy cfg * build: use reclient in appveyor * build: update WOA job too * build: force another build * build: do not checkout reclient
This commit is contained in:
Samuel Attard
GitHub
parent
c184b93fc5
commit
3afb012ad1
14 changed files with 301 additions and 148 deletions
144
patches/reclient-configs/fix_add_python_remote_wrapper.patch
Normal file
144
patches/reclient-configs/fix_add_python_remote_wrapper.patch
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Samuel Attard <samuel.r.attard@gmail.com>
|
||||
Date: Thu, 28 Dec 2023 16:47:13 +1300
|
||||
Subject: fix: add python_remote_wrapper
|
||||
|
||||
Our RBE cluster struggles to find python3 but bash finds it when spawned
|
||||
so we wrap all python remote execs with a helper bash script that just
|
||||
runs the underlying command.
|
||||
|
||||
diff --git a/configure_reclient.py b/configure_reclient.py
|
||||
index 5948be95b6b19e6af4f9a9503c18ce2bebf63f2d..a157ceeb155987f6ab9b11d3cef8c2c9894e0436 100755
|
||||
--- a/configure_reclient.py
|
||||
+++ b/configure_reclient.py
|
||||
@@ -21,6 +21,7 @@ import os
|
||||
import re
|
||||
import runpy
|
||||
import shutil
|
||||
+import stat
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -109,6 +110,8 @@ class ReclientConfigurator:
|
||||
self.download_linux_clang_toolchain()
|
||||
self.generate_clang_remote_wrapper()
|
||||
|
||||
+ self.generate_python_remote_wrapper()
|
||||
+
|
||||
# Reproxy config includes auth and network-related parameters.
|
||||
self.generate_reproxy_cfg()
|
||||
# Rewrapper configs describe how different tools should be run remotely.
|
||||
@@ -199,6 +202,32 @@ class ReclientConfigurator:
|
||||
(f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/'
|
||||
'clang_remote_wrapper'), clang_remote_wrapper)
|
||||
|
||||
+ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/'
|
||||
+ 'clang_remote_wrapper'))
|
||||
+
|
||||
+ @staticmethod
|
||||
+ def generate_python_remote_wrapper():
|
||||
+ # Load python remote wrapper template.
|
||||
+ template_file = (f'{Paths.script_dir}/python/'
|
||||
+ 'python_remote_wrapper.template')
|
||||
+ python_remote_wrapper_template = FileUtils.read_text_file(template_file)
|
||||
+
|
||||
+ # Variables to set in the template.
|
||||
+ template_vars = {
|
||||
+ }
|
||||
+
|
||||
+ # Substitute variables into the template.
|
||||
+ python_remote_wrapper = ShellTemplate(
|
||||
+ python_remote_wrapper_template).substitute(template_vars)
|
||||
+
|
||||
+ # Write the python remote wrapper.
|
||||
+ FileUtils.write_text_file(
|
||||
+ (f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
|
||||
+ 'python_remote_wrapper'), python_remote_wrapper)
|
||||
+
|
||||
+ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
|
||||
+ 'python_remote_wrapper'))
|
||||
+
|
||||
def generate_reproxy_cfg(self):
|
||||
# Load Chromium config template and remove everything starting with $
|
||||
# symbol on each line.
|
||||
@@ -536,6 +565,11 @@ class FileUtils:
|
||||
|
||||
shutil.move(filepath_new, filepath)
|
||||
|
||||
+ @classmethod
|
||||
+ def chmod_x(cls, filepath):
|
||||
+ st = os.stat(filepath)
|
||||
+ os.chmod(filepath, st.st_mode | stat.S_IEXEC)
|
||||
+
|
||||
@classmethod
|
||||
def create_generated_header(cls, source_files):
|
||||
if not isinstance(source_files, (list, tuple)):
|
||||
diff --git a/python/python_remote_wrapper.template b/python/python_remote_wrapper.template
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..54817e4f6f9e3cb2f1e7ea1317fa8fefdf7a7da5
|
||||
--- /dev/null
|
||||
+++ b/python/python_remote_wrapper.template
|
||||
@@ -0,0 +1,29 @@
|
||||
+#!/bin/bash
|
||||
+# Copyright (c) 2023 Contributors to the reclient-configs project. All rights reserved.
|
||||
+#
|
||||
+# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
+# you may not use this file except in compliance with the License.
|
||||
+# You may obtain a copy of the License at
|
||||
+#
|
||||
+# http://www.apache.org/licenses/LICENSE-2.0
|
||||
+#
|
||||
+# Unless required by applicable law or agreed to in writing, software
|
||||
+# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
+# See the License for the specific language governing permissions and
|
||||
+# limitations under the License.
|
||||
+
|
||||
+# AUTOGENERATED FILE - DO NOT EDIT
|
||||
+# Generated by:
|
||||
+# {configurator_dir}/configure_reclient.py
|
||||
+# To edit update:
|
||||
+# {configurator_dir}/python/python_remote_wrapper.template
|
||||
+# And rerun configurator.
|
||||
+
|
||||
+# WARNING: This file is a part of reclient action inputs. Any modification will
|
||||
+# invalidate remote cache.
|
||||
+
|
||||
+set -e
|
||||
+
|
||||
+# Launch
|
||||
+"$1" "${@:2}"
|
||||
diff --git a/python/rewrapper_linux.cfg b/python/rewrapper_linux.cfg
|
||||
index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644
|
||||
--- a/python/rewrapper_linux.cfg
|
||||
+++ b/python/rewrapper_linux.cfg
|
||||
@@ -13,3 +13,6 @@
|
||||
# limitations under the License.
|
||||
|
||||
# This config is merged with Chromium config. See README.md.
|
||||
+
|
||||
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
\ No newline at end of file
|
||||
diff --git a/python/rewrapper_mac.cfg b/python/rewrapper_mac.cfg
|
||||
index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644
|
||||
--- a/python/rewrapper_mac.cfg
|
||||
+++ b/python/rewrapper_mac.cfg
|
||||
@@ -13,3 +13,6 @@
|
||||
# limitations under the License.
|
||||
|
||||
# This config is merged with Chromium config. See README.md.
|
||||
+
|
||||
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
\ No newline at end of file
|
||||
diff --git a/python/rewrapper_windows.cfg b/python/rewrapper_windows.cfg
|
||||
index 7ff80607546455eb7c5f16a410770d18949cd7f5..ce19dc95094d87b494e4b89a2bf1a368ace567a2 100644
|
||||
--- a/python/rewrapper_windows.cfg
|
||||
+++ b/python/rewrapper_windows.cfg
|
||||
@@ -15,3 +15,5 @@
|
||||
# This config is merged with Chromium config. See README.md.
|
||||
|
||||
server_address=pipe://reproxy.pipe
|
||||
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
|
||||
Loading…
Add table
Add a link
Reference in a new issue