Merge branch 'emper-iwyu' into 'master'

[iwyu] Take load average into consideration on CI

See merge request i4/manycore/emper!151

.gitlab-ci.yml

@@ -54,7 +54,7 @@ fast-static-analysis:
 
 iwyu:
    stage: smoke-test
-   script: make iwyu
+   script: IWYU_TOOL="${CI_PROJECT_DIR}/tools/iwyu_tool.py" make iwyu
    variables:
        EMPER_IO: "true"
 

tools/check-iwyu

@@ -3,6 +3,10 @@ set -euo pipefail
 
 # Meson issue regarding iwyu integration: https://github.com/mesonbuild/meson/issues/2637
 
+echoerr() {
+	echo "${@}" 1>&2
+}
+
 # Pretty fancy method to get reliable the absolute path of a shell
 # script, *even if it is sourced*. Credits go to GreenFox on
 # stackoverflow: http://stackoverflow.com/a/12197518/194894
@@ -16,13 +20,11 @@ cd "`dirname "${SCRIPTDIR}"`" > /dev/null
 SCRIPTDIR="`pwd`";
 popd  > /dev/null
 
-set +u
-if [[ -n "${MESON_BUILD_ROOT}" ]]; then
+if [[ -v MESON_BUILD_ROOT ]]; then
 	MESON_BUILD_ROOT_SET=true
 else
 	MESON_BUILD_ROOT_SET=false
 fi
-set -u
 
 if ! ${MESON_BUILD_ROOT_SET}; then
 	ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
@@ -33,39 +35,62 @@ fi
 
 readonly IWYU_LOG="${MESON_BUILD_ROOT}/iwyu.log"
 
-NPROC=$(nproc)
+if [[ ! -v IWYU_TOOL ]]; then
+	for POSSIBLE_CMD in iwyu_tool iwyu-tool iwyu_tool.py; do
+		if command -v ${POSSIBLE_CMD} >/dev/null; then
+			IWYU_TOOL=${POSSIBLE_CMD}
+			break
+		fi
+	done
 
-for POSSIBLE_CMD in iwyu_tool iwyu-tool iwyu_tool.py; do
-	if command -v ${POSSIBLE_CMD} >/dev/null; then
-		IWYU_TOOL=${POSSIBLE_CMD}
-		break
+	if [[ ! -v IWYU_TOOL ]]; then
+		echoerr "iwyu_tool not found"
+		exit 1
 	fi
-done
+fi
 
-if [[ -z "${IWYU_TOOL}" ]]; then
-	echo "iwyu_tool not found"
-	exit 1
+NPROC=$(nproc)
+LOAD=$(python -c "print(${NPROC} * 1.5)")
+
+IWYU_TOOL_ARGS=(
+	-p "${MESON_BUILD_ROOT}"
+	--jobs "${NPROC}"
+)
+
+if "${IWYU_TOOL}" --help | grep -Fq -- '--load LOAD'; then
+	IWYU_TOOL_ARGS+=(--load "${LOAD}")
+else
+	# See https://github.com/include-what-you-use/include-what-you-use/pull/891
+	echo "WARNING: ${IWYU_TOOL} does not support --load"
 fi
 
-${IWYU_TOOL} -p "${MESON_BUILD_ROOT}" --jobs "${NPROC}" -- \
+${IWYU_TOOL} ${IWYU_TOOL_ARGS[@]} \
+			 -- \
 			 -Xiwyu --mapping_file="${MESON_SOURCE_ROOT}/iwyu-mappings.imp" \
 			 > "${IWYU_LOG}"
 
-# Sadly, iwyu_tool.py does not (yet) return an non-zero exit value if
+if [[ ! -s "${IWYU_LOG}" ]]; then
+	echoerr "${IWYU_LOG} is empty (or non existent)"
+	exit 1
+fi
+
+# Sadly, older iwyu_tool.py version do not return an non-zero exit value if
 # there are include issues, so we have to check the output manually.
 # See https://github.com/include-what-you-use/include-what-you-use/issues/790
-# Also note that the output contains "error: nos uch file or
+# Also note that the output contains "error: no such file or
 # directory: 'cc'" if ccache is used (which meson does by default if
 # ccache is available).
-# See https://github.com/include-what-you-use/include-what-you-use/issues/789
+# See
+# - https://github.com/include-what-you-use/include-what-you-use/issues/789
+# - https://github.com/include-what-you-use/include-what-you-use/commit/a7499e4a2b416592777cc4c33fca746d091af738
 ERROR_STRINGS=()
 ERROR_STRINGS+=("should add these lines:")
 ERROR_STRINGS+=("fatal error:")
 
 for ERROR_STRING in "${ERROR_STRINGS[@]}"; do
 	if grep -q "${ERROR_STRING}" "${IWYU_LOG}"; then
-		echo "IWYU found errors!"  
-		cat "${IWYU_LOG}"		   
+		echoerr "IWYU found errors!"
+		cat "${IWYU_LOG}"
 		exit 1
 	fi
 done

tools/iwyu_tool.py

+#!/usr/bin/env python
+
+##===--- iwyu_tool.py -----------------------------------------------------===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+""" Driver to consume a Clang compilation database and invoke IWYU.
+
+Example usage with CMake:
+
+  # Unix systems
+  $ mkdir build && cd build
+  $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
+  $ iwyu_tool.py -p .
+
+  # Windows systems
+  $ mkdir build && cd build
+  $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
+    -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+    -G Ninja ...
+  $ python iwyu_tool.py -p .
+
+See iwyu_tool.py -h for more details on command-line arguments.
+"""
+from __future__ import print_function
+import os
+import re
+import sys
+import json
+import time
+import shlex
+import argparse
+import tempfile
+import subprocess
+
+
+CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
+SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
+SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
+FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
+END_RE = re.compile(r'^---$')
+LINES_RE = re.compile(r'^- (.*?)  // lines ([0-9]+)-[0-9]+$')
+
+
+GENERAL, ADD, REMOVE, LIST = range(4)
+
+
+def clang_formatter(output):
+    """ Process iwyu's output into something clang-like. """
+    formatted = []
+
+    state = (GENERAL, None)
+    for line in output.splitlines():
+        match = CORRECT_RE.match(line)
+        if match:
+            formatted.append('%s:1:1: note: #includes/fwd-decls are correct' %
+                             match.groups(1))
+            continue
+        match = SHOULD_ADD_RE.match(line)
+        if match:
+            state = (ADD, match.group(1))
+            continue
+        match = SHOULD_REMOVE_RE.match(line)
+        if match:
+            state = (REMOVE, match.group(1))
+            continue
+        match = FULL_LIST_RE.match(line)
+        if match:
+            state = (LIST, match.group(1))
+        elif END_RE.match(line):
+            state = (GENERAL, None)
+        elif not line.strip():
+            continue
+        elif state[0] == GENERAL:
+            formatted.append(line)
+        elif state[0] == ADD:
+            formatted.append('%s:1:1: error: add the following line' % state[1])
+            formatted.append(line)
+        elif state[0] == REMOVE:
+            match = LINES_RE.match(line)
+            line_no = match.group(2) if match else '1'
+            formatted.append('%s:%s:1: error: remove the following line' %
+                             (state[1], line_no))
+            formatted.append(match.group(1))
+
+    return os.linesep.join(formatted)
+
+
+DEFAULT_FORMAT = 'iwyu'
+FORMATTERS = {
+    'iwyu': lambda output: output,
+    'clang': clang_formatter
+}
+
+
+if sys.platform.startswith('win'):
+    # Case-insensitive match on Windows
+    def normcase(s):
+        return s.lower()
+else:
+    def normcase(s):
+        return s
+
+
+def is_subpath_of(path, parent):
+    """ Return True if path is equal to or fully contained within parent.
+
+    Assumes both paths are canonicalized with os.path.realpath.
+    """
+    parent = normcase(parent)
+    path = normcase(path)
+
+    if path == parent:
+        return True
+
+    if not path.startswith(parent):
+        return False
+
+    # Now we know parent is a prefix of path, but they only share lineage if the
+    # difference between them starts with a path separator, e.g. /a/b/c/file
+    # is not a parent of /a/b/c/file.cpp, but /a/b/c and /a/b/c/ are.
+    parent = parent.rstrip(os.path.sep)
+    suffix = path[len(parent):]
+    return suffix.startswith(os.path.sep)
+
+
+def is_msvc_driver(compile_command):
+    """ Return True if compile_command matches an MSVC CL-style driver. """
+    compile_command = normcase(compile_command)
+
+    if compile_command.endswith('cl.exe'):
+        # Native MSVC compiler or clang-cl.exe
+        return True
+
+    if compile_command.endswith('clang-cl'):
+        # Cross clang-cl on non-Windows
+        return True
+
+    return False
+
+
+def win_split(cmdline):
+    """ Minimal implementation of shlex.split for Windows following
+    https://msdn.microsoft.com/en-us/library/windows/desktop/17w5ykft.aspx.
+    """
+    def split_iter(cmdline):
+        in_quotes = False
+        backslashes = 0
+        arg = ''
+        for c in cmdline:
+            if c == '\\':
+                # MSDN: Backslashes are interpreted literally, unless they
+                # immediately precede a double quotation mark.
+                # Buffer them until we know what comes next.
+                backslashes += 1
+            elif c == '"':
+                # Quotes can either be an escaped quote or the start of a quoted
+                # string. Paraphrasing MSDN:
+                # Before quotes, place one backslash in the arg for every pair
+                # of leading backslashes. If the number of backslashes is odd,
+                # retain the double quotation mark, otherwise interpret it as a
+                # string delimiter and switch state.
+                arg += '\\' * (backslashes // 2)
+                if backslashes % 2 == 1:
+                    arg += c
+                else:
+                    in_quotes = not in_quotes
+                backslashes = 0
+            elif c in (' ', '\t') and not in_quotes:
+                # MSDN: Arguments are delimited by white space, which is either
+                # a space or a tab [but only outside of a string].
+                # Flush backslashes and return arg bufferd so far, unless empty.
+                arg += '\\' * backslashes
+                if arg:
+                    yield arg
+                    arg = ''
+                backslashes = 0
+            else:
+                # Flush buffered backslashes and append.
+                arg += '\\' * backslashes
+                arg += c
+                backslashes = 0
+
+        if arg:
+            arg += '\\' * backslashes
+            yield arg
+
+    return list(split_iter(cmdline))
+
+
+def split_command(cmdstr):
+    """ Split a command string into a list, respecting shell quoting. """
+    if sys.platform.startswith('win'):
+        # shlex.split does not work for Windows command-lines, so special-case
+        # to our own implementation.
+        cmd = win_split(cmdstr)
+    else:
+        cmd = shlex.split(cmdstr)
+
+    return cmd
+
+
+def find_include_what_you_use():
+    """ Find IWYU executable and return its full pathname. """
+    if 'IWYU_BINARY' in os.environ:
+        return os.environ.get('IWYU_BINARY')
+
+    # TODO: Investigate using shutil.which when Python 2 has passed away.
+    executable_name = 'include-what-you-use'
+    if sys.platform.startswith('win'):
+        executable_name += '.exe'
+
+    search_path = [os.path.dirname(__file__)]
+    search_path += os.environ.get('PATH', '').split(os.pathsep)
+
+    for dirpath in search_path:
+        full = os.path.join(dirpath, executable_name)
+        if os.path.isfile(full):
+            return os.path.realpath(full)
+
+    return None
+
+
+IWYU_EXECUTABLE = find_include_what_you_use()
+
+
+class Process(object):
+    """ Manages an IWYU process in flight """
+    def __init__(self, proc, outfile):
+        self.proc = proc
+        self.outfile = outfile
+        self.output = None
+
+    def poll(self):
+        """ Return the exit code if the process has completed, None otherwise.
+        """
+        return self.proc.poll()
+
+    @property
+    def returncode(self):
+        return self.proc.returncode
+
+    def get_output(self):
+        """ Return stdout+stderr output of the process.
+
+        This call blocks until the process is complete, then returns the output.
+        """
+        if not self.output:
+            self.proc.wait()
+            self.outfile.seek(0)
+            self.output = self.outfile.read().decode("utf-8")
+            self.outfile.close()
+
+        return self.output
+
+    @classmethod
+    def start(cls, invocation):
+        """ Start a Process for the invocation and capture stdout+stderr. """
+        outfile = tempfile.TemporaryFile(prefix='iwyu')
+        process = subprocess.Popen(
+            invocation.command,
+            cwd=invocation.cwd,
+            stdout=outfile,
+            stderr=subprocess.STDOUT)
+        return cls(process, outfile)
+
+
+KNOWN_COMPILER_WRAPPERS=frozenset([
+    "ccache"
+])
+
+
+class Invocation(object):
+    """ Holds arguments of an IWYU invocation. """
+    def __init__(self, command, cwd):
+        self.command = command
+        self.cwd = cwd
+
+    def __str__(self):
+        return ' '.join(self.command)
+
+    @classmethod
+    def from_compile_command(cls, entry, extra_args):
+        """ Parse a JSON compilation database entry into new Invocation. """
+        if 'arguments' in entry:
+            # arguments is a command-line in list form.
+            command = entry['arguments']
+        elif 'command' in entry:
+            # command is a command-line in string form, split to list.
+            command = split_command(entry['command'])
+        else:
+            raise ValueError('Invalid compilation database entry: %s' % entry)
+
+        if command[0] in KNOWN_COMPILER_WRAPPERS:
+            # Remove the compiler wrapper from the command.
+            command = command[1:]
+
+        # Rewrite the compile command for IWYU
+        compile_command, compile_args = command[0], command[1:]
+        if is_msvc_driver(compile_command):
+            # If the compiler is cl-compatible, let IWYU be cl-compatible.
+            extra_args = ['--driver-mode=cl'] + extra_args
+
+        command = [IWYU_EXECUTABLE] + extra_args + compile_args
+        return cls(command, entry['directory'])
+
+    def start(self, verbose):
+        """ Run invocation and collect output. """
+        if verbose:
+            print('# %s' % self, file=sys.stderr)
+
+        return Process.start(self)
+
+
+def fixup_compilation_db(compilation_db):
+    """ Canonicalize paths in JSON compilation database. """
+    for entry in compilation_db:
+        # Convert relative paths to absolute ones if possible, based on the entry's directory.
+        if 'directory' in entry and not os.path.isabs(entry['file']):
+            entry['file'] = os.path.join(entry['directory'], entry['file'])
+
+        # Expand relative paths and symlinks
+        entry['file'] = os.path.realpath(entry['file'])
+
+    return compilation_db
+
+
+def slice_compilation_db(compilation_db, selection):
+    """ Return a new compilation database reduced to the paths in selection. """
+    if not selection:
+        return compilation_db
+
+    # Canonicalize selection paths to match compilation database.
+    selection = [os.path.realpath(p) for p in selection]
+
+    new_db = []
+    for path in selection:
+        if not os.path.exists(path):
+            print('warning: \'%s\' not found on disk.' % path, file=sys.stderr)
+            continue
+
+        found = [e for e in compilation_db if is_subpath_of(e['file'], path)]
+        if not found:
+            print('warning: \'%s\' not found in compilation database.' % path,
+                  file=sys.stderr)
+            continue
+
+        new_db.extend(found)
+
+    return new_db
+
+
+def execute(invocations, verbose, formatter, jobs, max_load_average=0):
+    """ Launch processes described by invocations. """
+    exit_code = 0
+    if jobs == 1:
+        for invocation in invocations:
+            proc = invocation.start(verbose)
+            print(formatter(proc.get_output()))
+            if proc.returncode != 2:
+                exit_code = 1
+        return exit_code
+
+    pending = []
+    while invocations or pending:
+        # Collect completed IWYU processes and print results.
+        complete = [proc for proc in pending if proc.poll() is not None]
+        for proc in complete:
+            pending.remove(proc)
+            print(formatter(proc.get_output()))
+            if proc.returncode != 2:
+                exit_code = 1
+
+        # Schedule new processes if there's room.
+        capacity = jobs - len(pending)
+
+        if max_load_average > 0:
+            one_min_load_average, _, _ = os.getloadavg()
+            load_capacity = max_load_average - one_min_load_average
+            if load_capacity < 0:
+                load_capacity = 0
+            if load_capacity < capacity:
+                capacity = int(load_capacity)
+                if not capacity and not pending:
+                    # Ensure there is at least one job running.
+                    capacity = 1
+
+        pending.extend(i.start(verbose) for i in invocations[:capacity])
+        invocations = invocations[capacity:]
+
+        # Yield CPU.
+        time.sleep(0.0001)
+    return exit_code
+
+
+def main(compilation_db_path, source_files, verbose, formatter, jobs,
+         max_load_average, extra_args):
+    """ Entry point. """
+
+    if not IWYU_EXECUTABLE:
+        print('error: include-what-you-use executable not found',
+              file=sys.stderr)
+        return 1
+
+    try:
+        if os.path.isdir(compilation_db_path):
+            compilation_db_path = os.path.join(compilation_db_path,
+                                               'compile_commands.json')
+
+        # Read compilation db from disk.
+        compilation_db_path = os.path.realpath(compilation_db_path)
+        with open(compilation_db_path, 'r') as fileobj:
+            compilation_db = json.load(fileobj)
+    except IOError as why:
+        print('error: failed to parse compilation database: %s' % why,
+              file=sys.stderr)
+        return 1
+
+    compilation_db = fixup_compilation_db(compilation_db)
+    compilation_db = slice_compilation_db(compilation_db, source_files)
+
+    # Transform compilation db entries into a list of IWYU invocations.
+    invocations = [
+        Invocation.from_compile_command(e, extra_args) for e in compilation_db
+    ]
+
+    return execute(invocations, verbose, formatter, jobs, max_load_average)
+
+
+def _bootstrap(sys_argv):
+    """ Parse arguments and dispatch to main(). """
+
+    # This hackery is necessary to add the forwarded IWYU args to the
+    # usage and help strings.
+    def customize_usage(parser):
+        """ Rewrite the parser's format_usage. """
+        original_format_usage = parser.format_usage
+        parser.format_usage = lambda: original_format_usage().rstrip() + \
+                              ' -- [<IWYU args>]' + os.linesep
+
+    def customize_help(parser):
+        """ Rewrite the parser's format_help. """
+        original_format_help = parser.format_help
+
+        def custom_help():
+            """ Customized help string, calls the adjusted format_usage. """
+            helpmsg = original_format_help()
+            helplines = helpmsg.splitlines()
+            helplines[0] = parser.format_usage().rstrip()
+            return os.linesep.join(helplines) + os.linesep
+
+        parser.format_help = custom_help
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(
+        description='Include-what-you-use compilation database driver.',
+        epilog='Assumes include-what-you-use is available on the PATH.')
+    customize_usage(parser)
+    customize_help(parser)
+
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Print IWYU commands')
+    parser.add_argument('-o', '--output-format', type=str,
+                        choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
+                        help='Output format (default: %s)' % DEFAULT_FORMAT)
+    parser.add_argument('-j', '--jobs', type=int, default=1,
+                        help='Number of concurrent subprocesses')
+    parser.add_argument('-l', '--load', type=float, default=0,
+                        help='Do not start new jobs if the 1min load average is greater than the provided value')
+    parser.add_argument('-p', metavar='<build-path>', required=True,
+                        help='Compilation database path', dest='dbpath')
+    parser.add_argument('source', nargs='*',
+                        help=('Zero or more source files (or directories) to '
+                              'run IWYU on. Defaults to all in compilation '
+                              'database.'))
+
+    def partition_args(argv):
+        """ Split around '--' into driver args and IWYU args. """
+        try:
+            double_dash = argv.index('--')
+            return argv[:double_dash], argv[double_dash+1:]
+        except ValueError:
+            return argv, []
+    argv, extra_args = partition_args(sys_argv[1:])
+    args = parser.parse_args(argv)
+
+    return main(args.dbpath, args.source, args.verbose,
+                FORMATTERS[args.output_format], args.jobs, args.load, extra_args)
+
+
+if __name__ == '__main__':
+    sys.exit(_bootstrap(sys.argv))