chore: deprecate apply-patches in favour of git-{import,export}-patches (#15300)

This commit is contained in:
Jeremy Apthorp 2018-10-24 11:24:11 -07:00 committed by GitHub
parent 4185efa08f
commit 335e9f68b7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
123 changed files with 4368 additions and 4780 deletions

View file

@ -1,69 +0,0 @@
#!/usr/bin/env python
import argparse
import os
import subprocess
import sys
from lib import git
from lib.patches import PatchesConfig
SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
PATCHES_DIR = os.path.join(SOURCE_ROOT, 'patches')
PATCHES_COMMON_DIR = os.path.join(PATCHES_DIR, 'common')
PATCHES_MIPS64EL_DIR = os.path.join(PATCHES_DIR, 'mips64el')
SRC = 'src'
SRC_DIR = os.path.join(SOURCE_ROOT, SRC)
def main():
args = parse_args()
project_root = os.path.abspath(args.project_root)
for folder in args.folders:
error = apply_patches_for_dir(folder, project_root, args.commit)
if error:
sys.stderr.write(error + '\n')
sys.stderr.flush()
return 1
return 0
def apply_patches_for_dir(directory, project_root, commit):
for root, dirs, files in os.walk(directory):
config = PatchesConfig.from_directory(root, project_root=project_root)
patches_list = config.get_patches_list()
if patches_list is None:
continue
(success, failed_patches) = patches_list.apply(commit=commit)
if not success:
patch_path = failed_patches[0].get_file_path()
return '{0} failed to apply'.format(os.path.basename(patch_path))
def parse_args():
parser = argparse.ArgumentParser(description='Apply all required patches.')
parser.add_argument('--commit', default=False, action='store_true',
help='Commit a patch after it has been applied')
parser.add_argument('--project-root', required=False, default=git.get_repo_root(os.path.abspath(__file__)),
help='Parent folder to resolve repos relative paths against')
parser.add_argument('-t', '--target_arch',
help='Target architecture')
parser.add_argument('-f', '--folder', dest='folders', help='Apply patches from this folder', nargs='*', default=[])
args = parser.parse_args()
if not args.folders:
args.folders.append(PATCHES_COMMON_DIR)
if args.target_arch == 'mips64el':
args.folders.append(PATCHES_MIPS64EL_DIR)
return args
if __name__ == '__main__':
sys.exit(main())

View file

@ -0,0 +1,36 @@
#!/usr/bin/env python
from lib import git
from lib.patches import patch_from_dir
patch_dirs = {
'src/electron/patches/common/chromium':
'src',
'src/electron/patches/common/boringssl':
'src/third_party/boringssl/src',
'src/electron/patches/common/ffmpeg':
'src/third_party/ffmpeg',
'src/electron/patches/common/skia':
'src/third_party/skia',
'src/electron/patches/common/v8':
'src/v8',
}
def apply_patches(dirs):
for patch_dir, repo in dirs.iteritems():
git.am(repo=repo, patch_data=patch_from_dir(patch_dir),
committer_name="Electron Scripts", committer_email="scripts@electron")
def main():
apply_patches(patch_dirs)
if __name__ == '__main__':
main()

126
script/git-export-patches Executable file
View file

@ -0,0 +1,126 @@
#!/usr/bin/env python
import argparse
import os
import re
import subprocess
import sys
def guess_base_commit(repo):
"""Guess which commit the patches might be based on"""
args = [
'git',
'-C',
repo,
'describe',
'--tags',
]
return subprocess.check_output(args).split('-')[0:2]
def format_patch(repo, since):
args = [
'git',
'-C',
repo,
'format-patch',
'--keep-subject',
'--no-stat',
'--stdout',
# The name of the parent commit object isn't useful information in this
# context, so zero it out to avoid needless patch-file churn.
'--zero-commit',
# Some versions of git print out different numbers of characters in the
# 'index' line of patches, so pass --full-index to get consistent
# behaviour.
'--full-index',
since
]
return subprocess.check_output(args)
def split_patches(patch_data):
"""Split a concatenated series of patches into N separate patches"""
patches = []
patch_start = re.compile('^From [0-9a-f]+ ')
for line in patch_data.splitlines():
if patch_start.match(line):
patches.append([])
patches[-1].append(line)
return patches
def munge_subject_to_filename(subject):
"""Derive a suitable filename from a commit's subject"""
if subject.endswith('.patch'):
subject = subject[:-6]
return re.sub(r'[^A-Za-z0-9-]+', '_', subject).strip('_').lower() + '.patch'
def get_file_name(patch):
"""Return the name of the file to which the patch should be written"""
for line in patch:
if line.startswith('Patch-Filename: '):
return line[len('Patch-Filename: '):]
# If no patch-filename header, munge the subject.
for line in patch:
if line.startswith('Subject: '):
return munge_subject_to_filename(line[len('Subject: '):])
def remove_patch_filename(patch):
"""Strip out the Patch-Filename trailer from a patch's message body"""
force_keep_next_line = False
for i, l in enumerate(patch):
is_patchfilename = l.startswith('Patch-Filename: ')
next_is_patchfilename = i < len(patch) - 1 and patch[i+1].startswith('Patch-Filename: ')
if not force_keep_next_line and (is_patchfilename or (next_is_patchfilename and len(l.rstrip()) == 0)):
pass # drop this line
else:
yield l
force_keep_next_line = l.startswith('Subject: ')
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output",
help="directory into which exported patches will be written")
parser.add_argument("patch_range",
nargs='?',
help="range of patches to export. Defaults to all commits since the "
"most recent tag or remote branch.")
args = parser.parse_args(argv)
repo = '.'
patch_range = args.patch_range
if patch_range is None:
patch_range, num_patches = guess_base_commit(repo)
sys.stderr.write("Exporting {} patches since {}\n".format(num_patches, patch_range))
patch_data = format_patch(repo, patch_range)
patches = split_patches(patch_data)
out_dir = args.output
try:
os.mkdir(out_dir)
except OSError:
pass
# remove old patches, so that deleted commits are correctly reflected in the
# patch files (as a removed file)
for p in os.listdir(out_dir):
if p.endswith('.patch'):
os.remove(os.path.join(out_dir, p))
with open(os.path.join(out_dir, '.patches'), 'w') as pl:
for patch in patches:
filename = get_file_name(patch)
with open(os.path.join(out_dir, filename), 'w') as f:
f.write('\n'.join(remove_patch_filename(patch)))
pl.write(filename + '\n')
if __name__ == '__main__':
main(sys.argv[1:])

28
script/git-import-patches Executable file
View file

@ -0,0 +1,28 @@
#!/usr/bin/env python
import argparse
import os
import sys
from lib import git
from lib.patches import patch_from_dir
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument("patch_dir",
help="directory containing patches to apply")
parser.add_argument("-3", "--3way",
action="store_true", dest='threeway',
help="use 3-way merge to resolve conflicts")
args = parser.parse_args(argv)
git.am(
repo='.',
patch_data=patch_from_dir(args.patch_dir),
threeway=args.threeway
)
if __name__ == '__main__':
main(sys.argv[1:])

View file

@ -9,8 +9,6 @@ structure, or make assumptions about the passed arguments or calls' outcomes.
import os
import subprocess
from lib.util import scoped_cwd
def is_repo_root(path):
path_exists = os.path.exists(path)
@ -42,8 +40,28 @@ def get_repo_root(path):
return get_repo_root(parent_path)
def am(repo, patch_data, threeway=False,
committer_name=None, committer_email=None):
args = []
if threeway:
args += ['--3way']
root_args = ['-C', repo]
if committer_name is not None:
root_args += ['-c', 'user.name=' + committer_name]
if committer_email is not None:
root_args += ['-c', 'user.email=' + committer_email]
command = ['git'] + root_args + ['am'] + args
proc = subprocess.Popen(
command,
stdin=subprocess.PIPE)
proc.communicate(patch_data)
if proc.returncode != 0:
raise RuntimeError("Command {} returned {}".format(command,
proc.returncode))
def apply_patch(repo, patch_path, directory=None, index=False, reverse=False):
args = ['git', 'apply',
args = ['git', '-C', repo, 'apply',
'--ignore-space-change',
'--ignore-whitespace',
'--whitespace', 'fix'
@ -56,35 +74,31 @@ def apply_patch(repo, patch_path, directory=None, index=False, reverse=False):
args += ['--reverse']
args += ['--', patch_path]
with scoped_cwd(repo):
return_code = subprocess.call(args)
applied_successfully = (return_code == 0)
return applied_successfully
return_code = subprocess.call(args)
applied_successfully = (return_code == 0)
return applied_successfully
def get_patch(repo, commit_hash):
args = ['git', 'diff-tree',
args = ['git', '-C', repo, 'diff-tree',
'-p',
commit_hash,
'--' # Explicitly tell Git `commit_hash` is a revision, not a path.
]
with scoped_cwd(repo):
return subprocess.check_output(args)
return subprocess.check_output(args)
def get_head_commit(repo):
args = ['git', 'rev-parse', 'HEAD']
args = ['git', '-C', repo, 'rev-parse', 'HEAD']
with scoped_cwd(repo):
return subprocess.check_output(args).strip()
return subprocess.check_output(args).strip()
def reset(repo):
args = ['git', 'reset']
args = ['git', '-C', repo, 'reset']
with scoped_cwd(repo):
subprocess.check_call(args)
subprocess.check_call(args)
def commit(repo, author, message):
@ -96,12 +110,11 @@ def commit(repo, author, message):
env['GIT_COMMITTER_NAME'] = 'Anonymous Committer'
env['GIT_COMMITTER_EMAIL'] = 'anonymous@electronjs.org'
args = ['git', 'commit',
args = ['git', '-C', repo, 'commit',
'--author', author,
'--message', message
]
with scoped_cwd(repo):
return_code = subprocess.call(args, env=env)
committed_successfully = (return_code == 0)
return committed_successfully
return_code = subprocess.call(args, env=env)
committed_successfully = (return_code == 0)
return committed_successfully

View file

@ -1,177 +1,27 @@
#!/usr/bin/env python
import os
import sys
from lib import git
SOURCE_ROOT = os.path.abspath(os.path.dirname(
os.path.dirname(os.path.dirname(__file__))))
VENDOR_DIR = os.path.join(SOURCE_ROOT, 'vendor')
PYYAML_LIB_DIR = os.path.join(VENDOR_DIR, 'pyyaml', 'lib')
sys.path.append(PYYAML_LIB_DIR)
import yaml #pylint: disable=wrong-import-position,wrong-import-order
class Patch:
def __init__(self, file_path, repo_path, paths_prefix=None,
author='Anonymous <anonymous@electronjs.org>', description=None):
self.author = author
self.description = description
self.file_path = file_path
self.paths_prefix = paths_prefix
self.repo_path = repo_path
def apply(self, reverse=False, commit=False, index=False):
# Add the change to index only if we're going to commit it later.
add_to_index = index or commit
patch_applied = git.apply_patch(self.repo_path, self.file_path,
directory=self.paths_prefix,
index=add_to_index, reverse=reverse)
if not patch_applied:
return False
if commit:
message = self.__get_commit_message(reverse)
patch_committed = git.commit(self.repo_path, author=self.author,
message=message)
return patch_committed
return True
def __get_commit_message(self, reverse):
message = self.description
if message is None:
message = os.path.basename(self.file_path)
if reverse:
message = 'Revert: ' + message
return message
def reverse(self):
return self.apply(reverse=True)
def get_file_path(self):
return self.file_path
class PatchesList:
"""A list of patches for a specific git repo."""
def __init__(self, repo_path, patches):
# TODO(alexeykuzmin): Make sure that all patches have the same repo_path.
self.repo_path = repo_path
self.patches = patches
def __len__(self):
return len(self.patches)
def apply(self, reverse=False, stop_on_error=True, commit=False):
all_patches_applied = True
failed_patches = []
for patch in self.patches:
# Even if `commit` is True we're not going to commit
# individual patches, it takes too much time in the Chromium repo.
# Applying all commits takes about 10 minutes (!) on a fast dev machine.
# Instead of it we are going only to add all changes to the index
# and commit them all at once later.
applied_successfully = patch.apply(reverse=reverse, index=commit,
commit=False)
if not applied_successfully:
all_patches_applied = False
failed_patches.append(patch)
should_stop_now = not applied_successfully and stop_on_error
if should_stop_now:
break
if commit and not all_patches_applied:
git.reset(self.repo_path)
if commit and all_patches_applied:
author = 'Electron Build Process <build@electronjs.org>'
message = 'Apply Electron patches'
git.commit(self.repo_path, author=author, message=message)
return (all_patches_applied, failed_patches)
def reverse(self, stop_on_error=True, commit=False):
return self.apply(reverse=True, stop_on_error=stop_on_error, commit=commit)
def read_patch(patch_dir, patch_filename):
"""Read a patch from |patch_dir/filename| and amend the commit message with
metadata about the patch file it came from."""
ret = []
with open(os.path.join(patch_dir, patch_filename)) as f:
for l in f.readlines():
if l.startswith('diff -'):
ret.append('Patch-Filename: {}\n'.format(patch_filename))
ret.append(l)
return ''.join(ret)
class PatchesConfig:
@staticmethod
def from_directory(dir_path, project_root, config_name='.patches.yaml'):
config_path = os.path.join(dir_path, config_name)
return PatchesConfig(config_path, project_root)
def patch_from_dir(patch_dir):
"""Read a directory of patches into a format suitable for passing to
'git am'"""
with open(os.path.join(patch_dir, ".patches")) as f:
patch_list = [l.rstrip('\n') for l in f.readlines()]
def __init__(self, config_path, project_root):
self.path = config_path
self.patches_list = None
self.project_root = project_root
def __parse(self):
contents = None
if os.path.isfile(self.path):
with open(self.path, 'r') as stream:
try:
contents = yaml.load(stream)
except yaml.YAMLError as e:
print(e)
return contents
def __create_patch(self, raw_data, base_directory, repo_path, paths_prefix):
author = raw_data['author']
if author is None: # Shouldn't actually happen.
author = 'Anonymous <anonymous@electronjs.org>'
relative_file_path = raw_data['file']
absolute_file_path = os.path.join(base_directory, relative_file_path)
# Use a patch file path as a commit summary
# and optional description as a commit body.
description = relative_file_path
if raw_data['description'] is not None:
description += '\n\n' + raw_data['description']
return Patch(absolute_file_path, repo_path, paths_prefix=paths_prefix,
author=author, description=description)
def __create_patches_list(self):
config_contents = self.__parse()
if config_contents is None:
return None
relative_repo_path = os.path.normpath(config_contents['repo'])
absolute_repo_path = os.path.join(self.project_root, relative_repo_path)
# If the 'repo' path is not really a git repository,
# then use that path as a prefix for patched files.
paths_prefix = None
if not git.is_repo_root(absolute_repo_path):
assert(git.is_repo_root(self.project_root))
absolute_repo_path = self.project_root
paths_prefix = relative_repo_path
patches_data = config_contents['patches']
base_directory = os.path.abspath(os.path.dirname(self.path))
patches = [self.__create_patch(data, base_directory, absolute_repo_path,
paths_prefix) for data in patches_data]
patches_list = PatchesList(repo_path=absolute_repo_path, patches=patches)
return patches_list
def get_patches_list(self):
if self.patches_list is not None:
return self.patches_list
patches_list = self.__create_patches_list()
self.patches_list = patches_list
return patches_list
return ''.join([
read_patch(patch_dir, patch_filename)
for patch_filename in patch_list
])

View file

@ -1,97 +0,0 @@
#!/usr/bin/env python
"""
Usage: patch -h
Use this script to selectively apply and reverse patches.
It is mostly useful to fix patches during upgrades to a new Chromium version.
"""
import argparse
import os
import subprocess
import sys
import lib.git as git
from lib.patches import Patch, PatchesList, PatchesConfig
def main():
args = parse_args()
directory = args.directory
force = args.force
patches = args.patch
project_root = args.project_root
repo = args.repo
reverse = args.reverse
if directory:
(success, failed_patches) = apply_patches_from_directory(
directory, project_root, force, reverse)
else:
(success, failed_patches) = apply_patches(repo, patches, force, reverse)
if success:
print 'Done: All patches applied.'
else:
failed_patches_paths = [p.get_file_path() for p in failed_patches]
print 'Error: {0} patch(es) failed:\n{1}'.format(
len(failed_patches), '\n'.join(failed_patches_paths))
return 0 if success else 1
def apply_patches(repo_path, patches_paths, force=False, reverse=False):
absolute_repo_path = os.path.abspath(repo_path)
patches = [Patch(os.path.abspath(patch_path), absolute_repo_path)
for patch_path in patches_paths]
patches_list = PatchesList(repo_path=absolute_repo_path, patches=patches)
stop_on_error = not force
return patches_list.apply(reverse=reverse, stop_on_error=stop_on_error)
def apply_patches_from_directory(directory, project_root,
force=False, reverse=False):
config = PatchesConfig.from_directory(directory, project_root=project_root)
patches_list = config.get_patches_list()
# Notify user if we didn't find any patch files.
if patches_list is None or len(patches_list) == 0:
print 'Warning: No patches found in the "{0}" folder.'.format(directory)
return (True, [])
# Then try to apply patches.
stop_on_error = not force
return patches_list.apply(reverse=reverse, stop_on_error=stop_on_error)
def parse_args():
parser = argparse.ArgumentParser(description='Apply patches to a git repo')
parser.add_argument('-f', '--force', default=False, action='store_true',
help='Do not stop on the first failed patch.')
parser.add_argument('--project-root', required=False,
default=git.get_repo_root(os.path.abspath(__file__)),
help='A folder to resolve repos relative paths against')
parser.add_argument('-R', '--reverse', default=False, action='store_true',
help='Apply patches in reverse.')
parser.add_argument('-r', '--repo', help='Path to a repository root folder.')
paths_group = parser.add_mutually_exclusive_group(required=True)
paths_group.add_argument('-d', '--directory',
help='Path to a directory with patches. \
If present, -p/--patch is ignored.')
paths_group.add_argument('-p', '--patch', nargs='*',
help='Path(s) to a patch file(s).')
args = parser.parse_args()
# Additional rules.
if args.patch is not None and args.repo is None:
parser.error("Repository path (-r/--repo) is required \
when you supply patches list.")
return args
if __name__ == '__main__':
sys.exit(main())