diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5490547b73dd56ef97cc9982fa24402833b43a31..9a8cb5b36a6e2a6740c0c13be0bd3d020e1525f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ fast-static-analysis:
iwyu:
stage: smoke-test
- script: make iwyu
+ script: IWYU_TOOL="${CI_PROJECT_DIR}/tools/iwyu_tool.py" make iwyu
variables:
EMPER_IO: "true"
diff --git a/tools/check-iwyu b/tools/check-iwyu
index 24f74423dce4f0aea3e4209e0e5c5cead1db32db..4efd45c649daf898e8ee982e933955200355d0b0 100755
--- a/tools/check-iwyu
+++ b/tools/check-iwyu
@@ -3,6 +3,10 @@ set -euo pipefail
# Meson issue regarding iwyu integration: https://github.com/mesonbuild/meson/issues/2637
+echoerr() {
+ echo "${@}" 1>&2
+}
+
# Pretty fancy method to get reliable the absolute path of a shell
# script, *even if it is sourced*. Credits go to GreenFox on
# stackoverflow: http://stackoverflow.com/a/12197518/194894
@@ -16,13 +20,11 @@ cd "`dirname "${SCRIPTDIR}"`" > /dev/null
SCRIPTDIR="`pwd`";
popd > /dev/null
-set +u
-if [[ -n "${MESON_BUILD_ROOT}" ]]; then
+if [[ -v MESON_BUILD_ROOT ]]; then
MESON_BUILD_ROOT_SET=true
else
MESON_BUILD_ROOT_SET=false
fi
-set -u
if ! ${MESON_BUILD_ROOT_SET}; then
ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
@@ -33,39 +35,62 @@ fi
readonly IWYU_LOG="${MESON_BUILD_ROOT}/iwyu.log"
-NPROC=$(nproc)
+if [[ ! -v IWYU_TOOL ]]; then
+ for POSSIBLE_CMD in iwyu_tool iwyu-tool iwyu_tool.py; do
+ if command -v ${POSSIBLE_CMD} >/dev/null; then
+ IWYU_TOOL=${POSSIBLE_CMD}
+ break
+ fi
+ done
-for POSSIBLE_CMD in iwyu_tool iwyu-tool iwyu_tool.py; do
- if command -v ${POSSIBLE_CMD} >/dev/null; then
- IWYU_TOOL=${POSSIBLE_CMD}
- break
+ if [[ ! -v IWYU_TOOL ]]; then
+ echoerr "iwyu_tool not found"
+ exit 1
fi
-done
+fi
-if [[ -z "${IWYU_TOOL}" ]]; then
- echo "iwyu_tool not found"
- exit 1
+NPROC=$(nproc)
+LOAD=$(python -c "print(${NPROC} * 1.5)")
+
+IWYU_TOOL_ARGS=(
+ -p "${MESON_BUILD_ROOT}"
+ --jobs "${NPROC}"
+)
+
+if "${IWYU_TOOL}" --help | grep -Fq -- '--load LOAD'; then
+ IWYU_TOOL_ARGS+=(--load "${LOAD}")
+else
+ # See https://github.com/include-what-you-use/include-what-you-use/pull/891
+ echo "WARNING: ${IWYU_TOOL} does not support --load"
fi
-${IWYU_TOOL} -p "${MESON_BUILD_ROOT}" --jobs "${NPROC}" -- \
+${IWYU_TOOL} ${IWYU_TOOL_ARGS[@]} \
+ -- \
-Xiwyu --mapping_file="${MESON_SOURCE_ROOT}/iwyu-mappings.imp" \
> "${IWYU_LOG}"
-# Sadly, iwyu_tool.py does not (yet) return an non-zero exit value if
+if [[ ! -s "${IWYU_LOG}" ]]; then
+ echoerr "${IWYU_LOG} is empty (or non existent)"
+ exit 1
+fi
+
+# Sadly, older iwyu_tool.py version do not return an non-zero exit value if
# there are include issues, so we have to check the output manually.
# See https://github.com/include-what-you-use/include-what-you-use/issues/790
-# Also note that the output contains "error: nos uch file or
+# Also note that the output contains "error: no such file or
# directory: 'cc'" if ccache is used (which meson does by default if
# ccache is available).
-# See https://github.com/include-what-you-use/include-what-you-use/issues/789
+# See
+# - https://github.com/include-what-you-use/include-what-you-use/issues/789
+# - https://github.com/include-what-you-use/include-what-you-use/commit/a7499e4a2b416592777cc4c33fca746d091af738
ERROR_STRINGS=()
ERROR_STRINGS+=("should add these lines:")
ERROR_STRINGS+=("fatal error:")
for ERROR_STRING in "${ERROR_STRINGS[@]}"; do
if grep -q "${ERROR_STRING}" "${IWYU_LOG}"; then
- echo "IWYU found errors!"
- cat "${IWYU_LOG}"
+ echoerr "IWYU found errors!"
+ cat "${IWYU_LOG}"
exit 1
fi
done
diff --git a/tools/iwyu_tool.py b/tools/iwyu_tool.py
new file mode 100755
index 0000000000000000000000000000000000000000..eaf0abc69445543cc039fde1cb6bc45f521319a3
--- /dev/null
+++ b/tools/iwyu_tool.py
@@ -0,0 +1,498 @@
+#!/usr/bin/env python
+
+##===--- iwyu_tool.py -----------------------------------------------------===##
+#
+# The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+""" Driver to consume a Clang compilation database and invoke IWYU.
+
+Example usage with CMake:
+
+ # Unix systems
+ $ mkdir build && cd build
+ $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
+ $ iwyu_tool.py -p .
+
+ # Windows systems
+ $ mkdir build && cd build
+ $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
+ -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
+ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+ -G Ninja ...
+ $ python iwyu_tool.py -p .
+
+See iwyu_tool.py -h for more details on command-line arguments.
+"""
+from __future__ import print_function
+import os
+import re
+import sys
+import json
+import time
+import shlex
+import argparse
+import tempfile
+import subprocess
+
+
+CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
+SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
+SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
+FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
+END_RE = re.compile(r'^---$')
+LINES_RE = re.compile(r'^- (.*?) // lines ([0-9]+)-[0-9]+$')
+
+
+GENERAL, ADD, REMOVE, LIST = range(4)
+
+
+def clang_formatter(output):
+ """ Process iwyu's output into something clang-like. """
+ formatted = []
+
+ state = (GENERAL, None)
+ for line in output.splitlines():
+ match = CORRECT_RE.match(line)
+ if match:
+ formatted.append('%s:1:1: note: #includes/fwd-decls are correct' %
+ match.groups(1))
+ continue
+ match = SHOULD_ADD_RE.match(line)
+ if match:
+ state = (ADD, match.group(1))
+ continue
+ match = SHOULD_REMOVE_RE.match(line)
+ if match:
+ state = (REMOVE, match.group(1))
+ continue
+ match = FULL_LIST_RE.match(line)
+ if match:
+ state = (LIST, match.group(1))
+ elif END_RE.match(line):
+ state = (GENERAL, None)
+ elif not line.strip():
+ continue
+ elif state[0] == GENERAL:
+ formatted.append(line)
+ elif state[0] == ADD:
+ formatted.append('%s:1:1: error: add the following line' % state[1])
+ formatted.append(line)
+ elif state[0] == REMOVE:
+ match = LINES_RE.match(line)
+ line_no = match.group(2) if match else '1'
+ formatted.append('%s:%s:1: error: remove the following line' %
+ (state[1], line_no))
+ formatted.append(match.group(1))
+
+ return os.linesep.join(formatted)
+
+
+DEFAULT_FORMAT = 'iwyu'
+FORMATTERS = {
+ 'iwyu': lambda output: output,
+ 'clang': clang_formatter
+}
+
+
+if sys.platform.startswith('win'):
+ # Case-insensitive match on Windows
+ def normcase(s):
+ return s.lower()
+else:
+ def normcase(s):
+ return s
+
+
+def is_subpath_of(path, parent):
+ """ Return True if path is equal to or fully contained within parent.
+
+ Assumes both paths are canonicalized with os.path.realpath.
+ """
+ parent = normcase(parent)
+ path = normcase(path)
+
+ if path == parent:
+ return True
+
+ if not path.startswith(parent):
+ return False
+
+ # Now we know parent is a prefix of path, but they only share lineage if the
+ # difference between them starts with a path separator, e.g. /a/b/c/file
+ # is not a parent of /a/b/c/file.cpp, but /a/b/c and /a/b/c/ are.
+ parent = parent.rstrip(os.path.sep)
+ suffix = path[len(parent):]
+ return suffix.startswith(os.path.sep)
+
+
+def is_msvc_driver(compile_command):
+ """ Return True if compile_command matches an MSVC CL-style driver. """
+ compile_command = normcase(compile_command)
+
+ if compile_command.endswith('cl.exe'):
+ # Native MSVC compiler or clang-cl.exe
+ return True
+
+ if compile_command.endswith('clang-cl'):
+ # Cross clang-cl on non-Windows
+ return True
+
+ return False
+
+
+def win_split(cmdline):
+ """ Minimal implementation of shlex.split for Windows following
+ https://msdn.microsoft.com/en-us/library/windows/desktop/17w5ykft.aspx.
+ """
+ def split_iter(cmdline):
+ in_quotes = False
+ backslashes = 0
+ arg = ''
+ for c in cmdline:
+ if c == '\\':
+ # MSDN: Backslashes are interpreted literally, unless they
+ # immediately precede a double quotation mark.
+ # Buffer them until we know what comes next.
+ backslashes += 1
+ elif c == '"':
+ # Quotes can either be an escaped quote or the start of a quoted
+ # string. Paraphrasing MSDN:
+ # Before quotes, place one backslash in the arg for every pair
+ # of leading backslashes. If the number of backslashes is odd,
+ # retain the double quotation mark, otherwise interpret it as a
+ # string delimiter and switch state.
+ arg += '\\' * (backslashes // 2)
+ if backslashes % 2 == 1:
+ arg += c
+ else:
+ in_quotes = not in_quotes
+ backslashes = 0
+ elif c in (' ', '\t') and not in_quotes:
+ # MSDN: Arguments are delimited by white space, which is either
+ # a space or a tab [but only outside of a string].
+ # Flush backslashes and return arg bufferd so far, unless empty.
+ arg += '\\' * backslashes
+ if arg:
+ yield arg
+ arg = ''
+ backslashes = 0
+ else:
+ # Flush buffered backslashes and append.
+ arg += '\\' * backslashes
+ arg += c
+ backslashes = 0
+
+ if arg:
+ arg += '\\' * backslashes
+ yield arg
+
+ return list(split_iter(cmdline))
+
+
+def split_command(cmdstr):
+ """ Split a command string into a list, respecting shell quoting. """
+ if sys.platform.startswith('win'):
+ # shlex.split does not work for Windows command-lines, so special-case
+ # to our own implementation.
+ cmd = win_split(cmdstr)
+ else:
+ cmd = shlex.split(cmdstr)
+
+ return cmd
+
+
+def find_include_what_you_use():
+ """ Find IWYU executable and return its full pathname. """
+ if 'IWYU_BINARY' in os.environ:
+ return os.environ.get('IWYU_BINARY')
+
+ # TODO: Investigate using shutil.which when Python 2 has passed away.
+ executable_name = 'include-what-you-use'
+ if sys.platform.startswith('win'):
+ executable_name += '.exe'
+
+ search_path = [os.path.dirname(__file__)]
+ search_path += os.environ.get('PATH', '').split(os.pathsep)
+
+ for dirpath in search_path:
+ full = os.path.join(dirpath, executable_name)
+ if os.path.isfile(full):
+ return os.path.realpath(full)
+
+ return None
+
+
+IWYU_EXECUTABLE = find_include_what_you_use()
+
+
+class Process(object):
+ """ Manages an IWYU process in flight """
+ def __init__(self, proc, outfile):
+ self.proc = proc
+ self.outfile = outfile
+ self.output = None
+
+ def poll(self):
+ """ Return the exit code if the process has completed, None otherwise.
+ """
+ return self.proc.poll()
+
+ @property
+ def returncode(self):
+ return self.proc.returncode
+
+ def get_output(self):
+ """ Return stdout+stderr output of the process.
+
+ This call blocks until the process is complete, then returns the output.
+ """
+ if not self.output:
+ self.proc.wait()
+ self.outfile.seek(0)
+ self.output = self.outfile.read().decode("utf-8")
+ self.outfile.close()
+
+ return self.output
+
+ @classmethod
+ def start(cls, invocation):
+ """ Start a Process for the invocation and capture stdout+stderr. """
+ outfile = tempfile.TemporaryFile(prefix='iwyu')
+ process = subprocess.Popen(
+ invocation.command,
+ cwd=invocation.cwd,
+ stdout=outfile,
+ stderr=subprocess.STDOUT)
+ return cls(process, outfile)
+
+
+KNOWN_COMPILER_WRAPPERS=frozenset([
+ "ccache"
+])
+
+
+class Invocation(object):
+ """ Holds arguments of an IWYU invocation. """
+ def __init__(self, command, cwd):
+ self.command = command
+ self.cwd = cwd
+
+ def __str__(self):
+ return ' '.join(self.command)
+
+ @classmethod
+ def from_compile_command(cls, entry, extra_args):
+ """ Parse a JSON compilation database entry into new Invocation. """
+ if 'arguments' in entry:
+ # arguments is a command-line in list form.
+ command = entry['arguments']
+ elif 'command' in entry:
+ # command is a command-line in string form, split to list.
+ command = split_command(entry['command'])
+ else:
+ raise ValueError('Invalid compilation database entry: %s' % entry)
+
+ if command[0] in KNOWN_COMPILER_WRAPPERS:
+ # Remove the compiler wrapper from the command.
+ command = command[1:]
+
+ # Rewrite the compile command for IWYU
+ compile_command, compile_args = command[0], command[1:]
+ if is_msvc_driver(compile_command):
+ # If the compiler is cl-compatible, let IWYU be cl-compatible.
+ extra_args = ['--driver-mode=cl'] + extra_args
+
+ command = [IWYU_EXECUTABLE] + extra_args + compile_args
+ return cls(command, entry['directory'])
+
+ def start(self, verbose):
+ """ Run invocation and collect output. """
+ if verbose:
+ print('# %s' % self, file=sys.stderr)
+
+ return Process.start(self)
+
+
+def fixup_compilation_db(compilation_db):
+ """ Canonicalize paths in JSON compilation database. """
+ for entry in compilation_db:
+ # Convert relative paths to absolute ones if possible, based on the entry's directory.
+ if 'directory' in entry and not os.path.isabs(entry['file']):
+ entry['file'] = os.path.join(entry['directory'], entry['file'])
+
+ # Expand relative paths and symlinks
+ entry['file'] = os.path.realpath(entry['file'])
+
+ return compilation_db
+
+
+def slice_compilation_db(compilation_db, selection):
+ """ Return a new compilation database reduced to the paths in selection. """
+ if not selection:
+ return compilation_db
+
+ # Canonicalize selection paths to match compilation database.
+ selection = [os.path.realpath(p) for p in selection]
+
+ new_db = []
+ for path in selection:
+ if not os.path.exists(path):
+ print('warning: \'%s\' not found on disk.' % path, file=sys.stderr)
+ continue
+
+ found = [e for e in compilation_db if is_subpath_of(e['file'], path)]
+ if not found:
+ print('warning: \'%s\' not found in compilation database.' % path,
+ file=sys.stderr)
+ continue
+
+ new_db.extend(found)
+
+ return new_db
+
+
+def execute(invocations, verbose, formatter, jobs, max_load_average=0):
+ """ Launch processes described by invocations. """
+ exit_code = 0
+ if jobs == 1:
+ for invocation in invocations:
+ proc = invocation.start(verbose)
+ print(formatter(proc.get_output()))
+ if proc.returncode != 2:
+ exit_code = 1
+ return exit_code
+
+ pending = []
+ while invocations or pending:
+ # Collect completed IWYU processes and print results.
+ complete = [proc for proc in pending if proc.poll() is not None]
+ for proc in complete:
+ pending.remove(proc)
+ print(formatter(proc.get_output()))
+ if proc.returncode != 2:
+ exit_code = 1
+
+ # Schedule new processes if there's room.
+ capacity = jobs - len(pending)
+
+ if max_load_average > 0:
+ one_min_load_average, _, _ = os.getloadavg()
+ load_capacity = max_load_average - one_min_load_average
+ if load_capacity < 0:
+ load_capacity = 0
+ if load_capacity < capacity:
+ capacity = int(load_capacity)
+ if not capacity and not pending:
+ # Ensure there is at least one job running.
+ capacity = 1
+
+ pending.extend(i.start(verbose) for i in invocations[:capacity])
+ invocations = invocations[capacity:]
+
+ # Yield CPU.
+ time.sleep(0.0001)
+ return exit_code
+
+
+def main(compilation_db_path, source_files, verbose, formatter, jobs,
+ max_load_average, extra_args):
+ """ Entry point. """
+
+ if not IWYU_EXECUTABLE:
+ print('error: include-what-you-use executable not found',
+ file=sys.stderr)
+ return 1
+
+ try:
+ if os.path.isdir(compilation_db_path):
+ compilation_db_path = os.path.join(compilation_db_path,
+ 'compile_commands.json')
+
+ # Read compilation db from disk.
+ compilation_db_path = os.path.realpath(compilation_db_path)
+ with open(compilation_db_path, 'r') as fileobj:
+ compilation_db = json.load(fileobj)
+ except IOError as why:
+ print('error: failed to parse compilation database: %s' % why,
+ file=sys.stderr)
+ return 1
+
+ compilation_db = fixup_compilation_db(compilation_db)
+ compilation_db = slice_compilation_db(compilation_db, source_files)
+
+ # Transform compilation db entries into a list of IWYU invocations.
+ invocations = [
+ Invocation.from_compile_command(e, extra_args) for e in compilation_db
+ ]
+
+ return execute(invocations, verbose, formatter, jobs, max_load_average)
+
+
+def _bootstrap(sys_argv):
+ """ Parse arguments and dispatch to main(). """
+
+ # This hackery is necessary to add the forwarded IWYU args to the
+ # usage and help strings.
+ def customize_usage(parser):
+ """ Rewrite the parser's format_usage. """
+ original_format_usage = parser.format_usage
+ parser.format_usage = lambda: original_format_usage().rstrip() + \
+ ' -- [<IWYU args>]' + os.linesep
+
+ def customize_help(parser):
+ """ Rewrite the parser's format_help. """
+ original_format_help = parser.format_help
+
+ def custom_help():
+ """ Customized help string, calls the adjusted format_usage. """
+ helpmsg = original_format_help()
+ helplines = helpmsg.splitlines()
+ helplines[0] = parser.format_usage().rstrip()
+ return os.linesep.join(helplines) + os.linesep
+
+ parser.format_help = custom_help
+
+ # Parse arguments.
+ parser = argparse.ArgumentParser(
+ description='Include-what-you-use compilation database driver.',
+ epilog='Assumes include-what-you-use is available on the PATH.')
+ customize_usage(parser)
+ customize_help(parser)
+
+ parser.add_argument('-v', '--verbose', action='store_true',
+ help='Print IWYU commands')
+ parser.add_argument('-o', '--output-format', type=str,
+ choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
+ help='Output format (default: %s)' % DEFAULT_FORMAT)
+ parser.add_argument('-j', '--jobs', type=int, default=1,
+ help='Number of concurrent subprocesses')
+ parser.add_argument('-l', '--load', type=float, default=0,
+ help='Do not start new jobs if the 1min load average is greater than the provided value')
+ parser.add_argument('-p', metavar='<build-path>', required=True,
+ help='Compilation database path', dest='dbpath')
+ parser.add_argument('source', nargs='*',
+ help=('Zero or more source files (or directories) to '
+ 'run IWYU on. Defaults to all in compilation '
+ 'database.'))
+
+ def partition_args(argv):
+ """ Split around '--' into driver args and IWYU args. """
+ try:
+ double_dash = argv.index('--')
+ return argv[:double_dash], argv[double_dash+1:]
+ except ValueError:
+ return argv, []
+ argv, extra_args = partition_args(sys_argv[1:])
+ args = parser.parse_args(argv)
+
+ return main(args.dbpath, args.source, args.verbose,
+ FORMATTERS[args.output_format], args.jobs, args.load, extra_args)
+
+
+if __name__ == '__main__':
+ sys.exit(_bootstrap(sys.argv))