#!/usr/bin/env python3 """A wrapper script around clang-format, suitable for linting multiple files and to use for continuous integration. This is an alternative API for the clang-format command line. It runs over multiple files and directories in parallel. A diff output is produced and a sensible exit code is returned. """ import argparse import codecs import difflib import errno import fnmatch import io import multiprocessing import os import posixpath import signal import subprocess import sys import traceback import tempfile from functools import partial from lib.util import get_buildtools_executable DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx,mm' DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' class ExitStatus: SUCCESS = 0 DIFF = 1 TROUBLE = 2 def excludes_from_file(ignore_file): excludes = [] try: with io.open(ignore_file, 'r', encoding='utf-8') as f: for line in f: if line.startswith('#'): continue pattern = line.rstrip() if not pattern: continue excludes.append(pattern) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return excludes def list_files(files, recursive=False, extensions=None, exclude=None): if extensions is None: extensions = [] if exclude is None: exclude = [] out = [] for f in files: if recursive and os.path.isdir(f): for dirpath, dnames, fnames in os.walk(f): fpaths = [os.path.join(dirpath, fname) for fname in fnames] for pattern in exclude: dnames[:] = [ x for x in dnames if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) ] fpaths = [ x for x in fpaths if not fnmatch.fnmatch(x, pattern) ] for fp in fpaths: ext = os.path.splitext(fp)[1][1:] if ext in extensions: out.append(fp) else: ext = os.path.splitext(f)[1][1:] if ext in extensions: out.append(f) return out def make_diff(diff_file, original, reformatted): return list( difflib.unified_diff( original, reformatted, fromfile=f'a/{diff_file}', tofile=f'b/{diff_file}', n=3)) class DiffError(Exception): def __init__(self, message, errs=None): super().__init__(message) self.errs = errs or [] class UnexpectedError(Exception): def __init__(self, message, exc=None): super().__init__(message) self.formatted_traceback = traceback.format_exc() self.exc = exc def run_clang_format_diff_wrapper(args, file_name): try: ret = run_clang_format_diff(args, file_name) return ret except DiffError: raise except Exception as e: # pylint: disable=W0707 raise UnexpectedError(f'{file_name}: {e.__class__.__name__}: {e}', e) def run_clang_format_diff(args, file_name): try: with io.open(file_name, 'r', encoding='utf-8') as f: original = f.readlines() except IOError as exc: # pylint: disable=W0707 raise DiffError(str(exc)) invocation = [args.clang_format_executable, file_name] if args.fix: invocation.append('-i') if args.style: invocation.extend(['--style', args.style]) if args.dry_run: print(" ".join(invocation)) return [], [] try: with subprocess.Popen(' '.join(invocation), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) as proc: outs = list(proc.stdout.readlines()) errs = list(proc.stderr.readlines()) proc.wait() if proc.returncode: code = proc.returncode msg = f"clang-format exited with code {code}: '{file_name}'" raise DiffError(msg, errs) except OSError as exc: # pylint: disable=raise-missing-from cmd = subprocess.list2cmdline(invocation) raise DiffError(f"Command '{cmd}' failed to start: {exc}") if args.fix: return None, errs if sys.platform == 'win32': file_name = file_name.replace(os.sep, posixpath.sep) return make_diff(file_name, original, outs), errs def bold_red(s): return '\x1b[1m\x1b[31m' + s + '\x1b[0m' def colorize(diff_lines): def bold(s): return '\x1b[1m' + s + '\x1b[0m' def cyan(s): return '\x1b[36m' + s + '\x1b[0m' def green(s): return '\x1b[32m' + s + '\x1b[0m' def red(s): return '\x1b[31m' + s + '\x1b[0m' for line in diff_lines: if line[:4] in ['--- ', '+++ ']: yield bold(line) elif line.startswith('@@ '): yield cyan(line) elif line.startswith('+'): yield green(line) elif line.startswith('-'): yield red(line) else: yield line def print_diff(diff_lines, use_color): if use_color: diff_lines = colorize(diff_lines) if sys.version_info[0] < 3: sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) else: sys.stdout.writelines(diff_lines) def print_trouble(prog, message, use_colors): error_text = 'error:' if use_colors: error_text = bold_red(error_text) print(f"{prog}: {error_text} {message}", file=sys.stderr) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--clang-format-executable', metavar='EXECUTABLE', help='path to the clang-format executable', default=get_buildtools_executable('clang-format')) parser.add_argument( '--extensions', help='comma-separated list of file extensions' f' (default: {DEFAULT_EXTENSIONS})', default=DEFAULT_EXTENSIONS) parser.add_argument( '--fix', help='if specified, reformat files in-place', action='store_true') parser.add_argument( '-r', '--recursive', action='store_true', help='run recursively over directories') parser.add_argument( '-d', '--dry-run', action='store_true', help='just print the list of files') parser.add_argument('files', metavar='file', nargs='+') parser.add_argument( '-q', '--quiet', action='store_true') parser.add_argument( '-c', '--changed', action='store_true', help='only run on changed files') parser.add_argument( '-j', metavar='N', type=int, default=0, help='run N clang-format jobs in parallel' ' (default number of cpus + 1)') parser.add_argument( '--color', default='auto', choices=['auto', 'always', 'never'], help='show colored diff (default: auto)') parser.add_argument( '-e', '--exclude', metavar='PATTERN', action='append', default=[], help='exclude paths matching the given glob-like pattern(s)' ' from recursive search') parser.add_argument( '--style', help='formatting style to apply ' '(LLVM/Google/Chromium/Mozilla/WebKit)') args = parser.parse_args() # use default signal handling, like diff return SIGINT value on ^C # https://bugs.python.org/issue14229#msg156446 signal.signal(signal.SIGINT, signal.SIG_DFL) try: signal.SIGPIPE except AttributeError: # compatibility, SIGPIPE does not exist on Windows pass else: signal.signal(signal.SIGPIPE, signal.SIG_DFL) colored_stdout = False colored_stderr = False if args.color == 'always': colored_stdout = True colored_stderr = True elif args.color == 'auto': colored_stdout = sys.stdout.isatty() colored_stderr = sys.stderr.isatty() retcode = ExitStatus.SUCCESS parse_files = [] if args.changed: with subprocess.Popen( "git diff --name-only --cached", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, universal_newlines=True ) as child: for line in child.communicate()[0].split("\n"): file_name = line.rstrip() # don't check deleted files if os.path.isfile(file_name): parse_files.append(file_name) else: parse_files = args.files excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) excludes.extend(args.exclude) files = list_files( parse_files, recursive=args.recursive, exclude=excludes, extensions=args.extensions.split(',')) if not files: return ExitStatus.SUCCESS njobs = args.j if njobs == 0: njobs = multiprocessing.cpu_count() + 1 njobs = min(len(files), njobs) if not args.fix: # pylint: disable=consider-using-with patch_file = tempfile.NamedTemporaryFile(delete=False, prefix='electron-format-') if njobs == 1: # execute directly instead of in a pool, # less overhead, simpler stacktraces it = (run_clang_format_diff_wrapper(args, file) for file in files) pool = None else: # pylint: disable=consider-using-with pool = multiprocessing.Pool(njobs) it = pool.imap_unordered( partial(run_clang_format_diff_wrapper, args), files) while True: try: outs, errs = next(it) except StopIteration: break except DiffError as e: print_trouble(parser.prog, str(e), use_colors=colored_stderr) retcode = ExitStatus.TROUBLE sys.stderr.writelines(e.errs) except UnexpectedError as e: print_trouble(parser.prog, str(e), use_colors=colored_stderr) sys.stderr.write(e.formatted_traceback) retcode = ExitStatus.TROUBLE # stop at the first unexpected error, # something could be very wrong, # don't process all files unnecessarily if pool: pool.terminate() break else: sys.stderr.writelines(errs) if outs == []: continue if not args.fix: if not args.quiet: print_diff(outs, use_color=colored_stdout) for line in outs: patch_file.write(line.encode('utf-8')) patch_file.write('\n'.encode('utf-8')) if retcode == ExitStatus.SUCCESS: retcode = ExitStatus.DIFF if not args.fix: if patch_file.tell() == 0: patch_file.close() os.unlink(patch_file.name) else: print( 'To patch these files, run:', f"$ git apply {patch_file.name}", sep='\n') filename=patch_file.name print(f"\nTo patch these files, run:\n$ git apply {filename}\n") return retcode if __name__ == '__main__': sys.exit(main())