3ac57e17d5* build: fix clang format path * fix: use gclient sync instead of gclient runhooks Refs https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5659261 * fix: use gclient sync instead of gclient runhooks in GHA --------- Co-authored-by: David Sanders <dsanders11@ucsbalum.com>
384 lines
11 KiB
Python
384 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""A wrapper script around clang-format, suitable for linting multiple files
|
|
and to use for continuous integration.
|
|
|
|
This is an alternative API for the clang-format command line.
|
|
It runs over multiple files and directories in parallel.
|
|
A diff output is produced and a sensible exit code is returned.
|
|
"""
|
|
|
|
import argparse
|
|
import codecs
|
|
import difflib
|
|
import errno
|
|
import fnmatch
|
|
import io
|
|
import multiprocessing
|
|
import os
|
|
import posixpath
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import traceback
|
|
import tempfile
|
|
|
|
from functools import partial
|
|
from lib.util import get_depot_tools_executable
|
|
|
|
DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,mm'
|
|
DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore'
|
|
|
|
class ExitStatus:
|
|
SUCCESS = 0
|
|
DIFF = 1
|
|
TROUBLE = 2
|
|
|
|
def excludes_from_file(ignore_file):
|
|
excludes = []
|
|
try:
|
|
with io.open(ignore_file, 'r', encoding='utf-8') as f:
|
|
for line in f:
|
|
if line.startswith('#'):
|
|
continue
|
|
pattern = line.rstrip()
|
|
if not pattern:
|
|
continue
|
|
excludes.append(pattern)
|
|
except EnvironmentError as e:
|
|
if e.errno != errno.ENOENT:
|
|
raise
|
|
return excludes
|
|
|
|
def list_files(files, recursive=False, extensions=None, exclude=None):
|
|
if extensions is None:
|
|
extensions = []
|
|
if exclude is None:
|
|
exclude = []
|
|
|
|
out = []
|
|
for f in files:
|
|
if recursive and os.path.isdir(f):
|
|
for dirpath, dnames, fnames in os.walk(f):
|
|
fpaths = [os.path.join(dirpath, fname) for fname in fnames]
|
|
for pattern in exclude:
|
|
dnames[:] = [
|
|
x for x in dnames
|
|
if
|
|
not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
|
|
]
|
|
fpaths = [
|
|
x for x in fpaths if not fnmatch.fnmatch(x, pattern)
|
|
]
|
|
for fp in fpaths:
|
|
ext = os.path.splitext(fp)[1][1:]
|
|
if ext in extensions:
|
|
out.append(fp)
|
|
else:
|
|
ext = os.path.splitext(f)[1][1:]
|
|
if ext in extensions:
|
|
out.append(f)
|
|
return out
|
|
|
|
|
|
def make_diff(diff_file, original, reformatted):
|
|
return list(
|
|
difflib.unified_diff(
|
|
original,
|
|
reformatted,
|
|
fromfile=f'a/{diff_file}',
|
|
tofile=f'b/{diff_file}',
|
|
n=3))
|
|
|
|
|
|
class DiffError(Exception):
|
|
def __init__(self, message, errs=None):
|
|
super().__init__(message)
|
|
self.errs = errs or []
|
|
|
|
|
|
class UnexpectedError(Exception):
|
|
def __init__(self, message, exc=None):
|
|
super().__init__(message)
|
|
self.formatted_traceback = traceback.format_exc()
|
|
self.exc = exc
|
|
|
|
|
|
def run_clang_format_diff_wrapper(args, file_name):
|
|
try:
|
|
ret = run_clang_format_diff(args, file_name)
|
|
return ret
|
|
except DiffError:
|
|
raise
|
|
except Exception as e:
|
|
# pylint: disable=W0707
|
|
raise UnexpectedError(f'{file_name}: {e.__class__.__name__}: {e}', e)
|
|
|
|
|
|
def run_clang_format_diff(args, file_name):
|
|
try:
|
|
with io.open(file_name, 'r', encoding='utf-8') as f:
|
|
original = f.readlines()
|
|
except IOError as exc:
|
|
# pylint: disable=W0707
|
|
raise DiffError(str(exc))
|
|
invocation = [args.clang_format_executable, file_name]
|
|
if args.fix:
|
|
invocation.append('-i')
|
|
if args.style:
|
|
invocation.extend(['--style', args.style])
|
|
if args.dry_run:
|
|
print(" ".join(invocation))
|
|
return [], []
|
|
try:
|
|
with subprocess.Popen(' '.join(invocation),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
shell=True) as proc:
|
|
outs = list(proc.stdout.readlines())
|
|
errs = list(proc.stderr.readlines())
|
|
proc.wait()
|
|
if proc.returncode:
|
|
code = proc.returncode
|
|
msg = f"clang-format exited with code {code}: '{file_name}'"
|
|
raise DiffError(msg, errs)
|
|
except OSError as exc:
|
|
# pylint: disable=raise-missing-from
|
|
cmd = subprocess.list2cmdline(invocation)
|
|
raise DiffError(f"Command '{cmd}' failed to start: {exc}")
|
|
if args.fix:
|
|
return None, errs
|
|
if sys.platform == 'win32':
|
|
file_name = file_name.replace(os.sep, posixpath.sep)
|
|
return make_diff(file_name, original, outs), errs
|
|
|
|
|
|
def bold_red(s):
|
|
return '\x1b[1m\x1b[31m' + s + '\x1b[0m'
|
|
|
|
|
|
def colorize(diff_lines):
|
|
def bold(s):
|
|
return '\x1b[1m' + s + '\x1b[0m'
|
|
|
|
def cyan(s):
|
|
return '\x1b[36m' + s + '\x1b[0m'
|
|
|
|
def green(s):
|
|
return '\x1b[32m' + s + '\x1b[0m'
|
|
|
|
def red(s):
|
|
return '\x1b[31m' + s + '\x1b[0m'
|
|
|
|
for line in diff_lines:
|
|
if line[:4] in ['--- ', '+++ ']:
|
|
yield bold(line)
|
|
elif line.startswith('@@ '):
|
|
yield cyan(line)
|
|
elif line.startswith('+'):
|
|
yield green(line)
|
|
elif line.startswith('-'):
|
|
yield red(line)
|
|
else:
|
|
yield line
|
|
|
|
|
|
def print_diff(diff_lines, use_color):
|
|
if use_color:
|
|
diff_lines = colorize(diff_lines)
|
|
if sys.version_info[0] < 3:
|
|
sys.stdout.writelines((l.encode('utf-8') for l in diff_lines))
|
|
else:
|
|
sys.stdout.writelines(diff_lines)
|
|
|
|
|
|
def print_trouble(prog, message, use_colors):
|
|
error_text = 'error:'
|
|
if use_colors:
|
|
error_text = bold_red(error_text)
|
|
print(f"{prog}: {error_text} {message}", file=sys.stderr)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
'--clang-format-executable',
|
|
metavar='EXECUTABLE',
|
|
help='path to the clang-format executable',
|
|
default=get_depot_tools_executable('clang-format'))
|
|
parser.add_argument(
|
|
'--extensions',
|
|
help='comma-separated list of file extensions'
|
|
f' (default: {DEFAULT_EXTENSIONS})',
|
|
default=DEFAULT_EXTENSIONS)
|
|
parser.add_argument(
|
|
'--fix',
|
|
help='if specified, reformat files in-place',
|
|
action='store_true')
|
|
parser.add_argument(
|
|
'-r',
|
|
'--recursive',
|
|
action='store_true',
|
|
help='run recursively over directories')
|
|
parser.add_argument(
|
|
'-d',
|
|
'--dry-run',
|
|
action='store_true',
|
|
help='just print the list of files')
|
|
parser.add_argument('files', metavar='file', nargs='+')
|
|
parser.add_argument(
|
|
'-q',
|
|
'--quiet',
|
|
action='store_true')
|
|
parser.add_argument(
|
|
'-c',
|
|
'--changed',
|
|
action='store_true',
|
|
help='only run on changed files')
|
|
parser.add_argument(
|
|
'-j',
|
|
metavar='N',
|
|
type=int,
|
|
default=0,
|
|
help='run N clang-format jobs in parallel'
|
|
' (default number of cpus + 1)')
|
|
parser.add_argument(
|
|
'--color',
|
|
default='auto',
|
|
choices=['auto', 'always', 'never'],
|
|
help='show colored diff (default: auto)')
|
|
parser.add_argument(
|
|
'-e',
|
|
'--exclude',
|
|
metavar='PATTERN',
|
|
action='append',
|
|
default=[],
|
|
help='exclude paths matching the given glob-like pattern(s)'
|
|
' from recursive search')
|
|
parser.add_argument(
|
|
'--style',
|
|
help='formatting style to apply '
|
|
'(LLVM/Google/Chromium/Mozilla/WebKit)')
|
|
|
|
args = parser.parse_args()
|
|
|
|
# use default signal handling, like diff return SIGINT value on ^C
|
|
# https://bugs.python.org/issue14229#msg156446
|
|
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
|
try:
|
|
signal.SIGPIPE
|
|
except AttributeError:
|
|
# compatibility, SIGPIPE does not exist on Windows
|
|
pass
|
|
else:
|
|
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
|
|
colored_stdout = False
|
|
colored_stderr = False
|
|
if args.color == 'always':
|
|
colored_stdout = True
|
|
colored_stderr = True
|
|
elif args.color == 'auto':
|
|
colored_stdout = sys.stdout.isatty()
|
|
colored_stderr = sys.stderr.isatty()
|
|
|
|
retcode = ExitStatus.SUCCESS
|
|
|
|
parse_files = []
|
|
if args.changed:
|
|
with subprocess.Popen(
|
|
"git diff --name-only --cached",
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
shell=True,
|
|
universal_newlines=True
|
|
) as child:
|
|
for line in child.communicate()[0].split("\n"):
|
|
file_name = line.rstrip()
|
|
# don't check deleted files
|
|
if os.path.isfile(file_name):
|
|
parse_files.append(file_name)
|
|
|
|
else:
|
|
parse_files = args.files
|
|
|
|
excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE)
|
|
excludes.extend(args.exclude)
|
|
|
|
files = list_files(
|
|
parse_files,
|
|
recursive=args.recursive,
|
|
exclude=excludes,
|
|
extensions=args.extensions.split(','))
|
|
|
|
if not files:
|
|
return ExitStatus.SUCCESS
|
|
|
|
njobs = args.j
|
|
if njobs == 0:
|
|
njobs = multiprocessing.cpu_count() + 1
|
|
njobs = min(len(files), njobs)
|
|
|
|
if not args.fix:
|
|
# pylint: disable=consider-using-with
|
|
patch_file = tempfile.NamedTemporaryFile(delete=False,
|
|
prefix='electron-format-')
|
|
|
|
if njobs == 1:
|
|
# execute directly instead of in a pool,
|
|
# less overhead, simpler stacktraces
|
|
it = (run_clang_format_diff_wrapper(args, file) for file in files)
|
|
pool = None
|
|
else:
|
|
# pylint: disable=consider-using-with
|
|
pool = multiprocessing.Pool(njobs)
|
|
it = pool.imap_unordered(
|
|
partial(run_clang_format_diff_wrapper, args), files)
|
|
while True:
|
|
try:
|
|
outs, errs = next(it)
|
|
except StopIteration:
|
|
break
|
|
except DiffError as e:
|
|
print_trouble(parser.prog, str(e), use_colors=colored_stderr)
|
|
retcode = ExitStatus.TROUBLE
|
|
sys.stderr.writelines(e.errs)
|
|
except UnexpectedError as e:
|
|
print_trouble(parser.prog, str(e), use_colors=colored_stderr)
|
|
sys.stderr.write(e.formatted_traceback)
|
|
retcode = ExitStatus.TROUBLE
|
|
# stop at the first unexpected error,
|
|
# something could be very wrong,
|
|
# don't process all files unnecessarily
|
|
if pool:
|
|
pool.terminate()
|
|
break
|
|
else:
|
|
sys.stderr.writelines(errs)
|
|
if outs == []:
|
|
continue
|
|
if not args.fix:
|
|
if not args.quiet:
|
|
print_diff(outs, use_color=colored_stdout)
|
|
for line in outs:
|
|
patch_file.write(line.encode('utf-8'))
|
|
patch_file.write('\n'.encode('utf-8'))
|
|
if retcode == ExitStatus.SUCCESS:
|
|
retcode = ExitStatus.DIFF
|
|
|
|
if not args.fix:
|
|
if patch_file.tell() == 0:
|
|
patch_file.close()
|
|
os.unlink(patch_file.name)
|
|
else:
|
|
print(
|
|
'To patch these files, run:',
|
|
f"$ git apply {patch_file.name}", sep='\n')
|
|
filename=patch_file.name
|
|
print(f"\nTo patch these files, run:\n$ git apply {filename}\n")
|
|
|
|
return retcode
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|