Merge pull request #6199 from mensinda/cmSysInc

cmake: Handle CMake system include dirs (closes #6079)

mesonbuild/backend/ninjabackend.py

@@ -334,6 +334,7 @@ int dummy;
         # Only overwrite the old build file after the new one has been
         # fully created.
         os.replace(tempfilename, outfilename)
+        mlog.cmd_ci_include(outfilename)  # For CI debugging
         self.generate_compdb()
 
     # http://clang.llvm.org/docs/JSONCompilationDatabase.html

mesonbuild/cmake/common.py

@@ -73,9 +73,10 @@ class CMakeFileGroup:
         tmp = []
         for i in self.includes:
             if isinstance(i, dict) and 'path' in i:
-                tmp += [i['path']]
-            elif isinstance(i, str):
+                i['isSystem'] = i.get('isSystem', False)
                 tmp += [i]
+            elif isinstance(i, str):
+                tmp += [{'path': i, 'isSystem': False}]
         self.includes = tmp
 
     def log(self) -> None:

mesonbuild/cmake/fileapi.py

@@ -14,6 +14,7 @@
 
 from .common import CMakeException, CMakeBuildFile, CMakeConfiguration
 from typing import Any, List, Tuple
+from .. import mlog
 import os
 import json
 import re
@@ -76,6 +77,7 @@ class CMakeFileAPI:
         debug_json = os.path.normpath(os.path.join(self.build_dir, '..', 'fileAPI.json'))
         with open(debug_json, 'w') as fp:
             json.dump(index, fp, indent=2)
+        mlog.cmd_ci_include(debug_json)
 
         # parse the JSON
         for i in index['objects']:
@@ -186,9 +188,7 @@ class CMakeFileAPI:
                     'language': cg.get('language', 'C'),
                     'isGenerated': None,  # Set later, flag is stored per source file
                     'sources': [],
-
-                    # TODO handle isSystem
-                    'includePath': [x.get('path', '') for x in cg.get('includes', [])],
+                    'includePath': cg.get('includes', []),
                 }
 
                 normal_src, generated_src, src_idx = parse_sources(cg, tgt)

mesonbuild/cmake/interpreter.py

@@ -157,6 +157,7 @@ class ConverterTarget:
         self.sources = []
         self.generated = []
         self.includes = []
+        self.sys_includes = []
         self.link_with = []
         self.object_libs = []
         self.compile_opts = {}
@@ -180,7 +181,8 @@ class ConverterTarget:
             self.compile_opts[lang] += [x for x in args if x not in self.compile_opts[lang]]
 
             # Handle include directories
-            self.includes += [x for x in i.includes if x not in self.includes]
+            self.includes += [x['path'] for x in i.includes if x not in self.includes and not x['isSystem']]
+            self.sys_includes += [x['path'] for x in i.includes if x not in self.sys_includes and x['isSystem']]
 
             # Add sources to the right array
             if i.is_generated:
@@ -295,6 +297,7 @@ class ConverterTarget:
 
         build_dir_rel = os.path.relpath(self.build_dir, os.path.join(self.env.get_build_dir(), subdir))
         self.includes = list(set([rel_path(x, True, False) for x in set(self.includes)] + [build_dir_rel]))
+        self.sys_includes = list(set([rel_path(x, True, False) for x in set(self.sys_includes)]))
         self.sources = [rel_path(x, False, False) for x in self.sources]
         self.generated = [rel_path(x, False, True) for x in self.generated]
 
@@ -303,6 +306,7 @@ class ConverterTarget:
 
         # Remove delete entries
         self.includes = [x for x in self.includes if x is not None]
+        self.sys_includes = [x for x in self.sys_includes if x is not None]
         self.sources = [x for x in self.sources if x is not None]
         self.generated = [x for x in self.generated if x is not None]
 
@@ -359,6 +363,7 @@ class ConverterTarget:
         mlog.log('  -- link_flags:     ', mlog.bold(str(self.link_flags)))
         mlog.log('  -- languages:      ', mlog.bold(str(self.languages)))
         mlog.log('  -- includes:       ', mlog.bold(str(self.includes)))
+        mlog.log('  -- sys_includes:   ', mlog.bold(str(self.sys_includes)))
         mlog.log('  -- sources:        ', mlog.bold(str(self.sources)))
         mlog.log('  -- generated:      ', mlog.bold(str(self.generated)))
         mlog.log('  -- pie:            ', mlog.bold('true' if self.pie else 'false'))
@@ -845,6 +850,8 @@ class CMakeInterpreter:
             base_name = str(tgt.name)
             base_name = base_name.replace('-', '_')
             inc_var = '{}_inc'.format(base_name)
+            dir_var = '{}_dir'.format(base_name)
+            sys_var = '{}_sys'.format(base_name)
             src_var = '{}_src'.format(base_name)
             dep_var = '{}_dep'.format(base_name)
             tgt_var = base_name
@@ -879,8 +886,10 @@ class CMakeInterpreter:
             }
 
             # Generate the function nodes
-            inc_node = assign(inc_var, function('include_directories', tgt.includes))
-            node_list = [inc_node]
+            dir_node = assign(dir_var, function('include_directories', tgt.includes))
+            sys_node = assign(sys_var, function('include_directories', tgt.sys_includes, {'is_system': True}))
+            inc_node = assign(inc_var, array([id_node(dir_var), id_node(sys_var)]))
+            node_list = [dir_node, sys_node, inc_node]
             if tgt_func == 'header_only':
                 del dep_kwargs['link_with']
                 dep_node = assign(dep_var, function('declare_dependency', kwargs=dep_kwargs))

mesonbuild/interpreter.py

@@ -2597,6 +2597,7 @@ external dependencies (including libraries) must go to "dependencies".''')
                     f.write(printer.result)
 
                 mlog.log('Build file:', meson_filename)
+                mlog.cmd_ci_include(meson_filename)
                 mlog.log()
 
             result = self._do_subproject_meson(dirname, subdir, default_options, kwargs, ast, cm_int.bs_files)

mesonbuild/mlog.py

@@ -56,6 +56,7 @@ log_timestamp_start = None   # type: Optional[float]
 log_fatal_warnings = False   # type: bool
 log_disable_stdout = False   # type: bool
 log_errors_only = False      # type: bool
+_in_ci = 'CI' in os.environ  # type: bool
 
 def disable() -> None:
     global log_disable_stdout
@@ -186,6 +187,15 @@ def debug(*args: Union[str, AnsiDecorator], **kwargs: Any) -> None:
         print(*arr, file=log_file, **kwargs)
         log_file.flush()
 
+def _debug_log_cmd(cmd: str, args: List[str]) -> None:
+    if not _in_ci:
+        return
+    args = ['"{}"'.format(x) for x in args]  # Quote all args, just in case
+    debug('!meson_ci!/{} {}'.format(cmd, ' '.join(args)))
+
+def cmd_ci_include(file: str) -> None:
+    _debug_log_cmd('ci_include', [file])
+
 def log(*args: Union[str, AnsiDecorator], is_error: bool = False,
         **kwargs: Any) -> None:
     global log_errors_only

run_project_tests.py

@@ -21,6 +21,7 @@ import subprocess
 import shutil
 import sys
 import signal
+import shlex
 from io import StringIO
 from ast import literal_eval
 from enum import Enum
@@ -57,12 +58,13 @@ class BuildStep(Enum):
 
 
 class TestResult:
-    def __init__(self, msg, step, stdo, stde, mlog, conftime=0, buildtime=0, testtime=0):
+    def __init__(self, msg, step, stdo, stde, mlog, cicmds, conftime=0, buildtime=0, testtime=0):
         self.msg = msg
         self.step = step
         self.stdo = stdo
         self.stde = stde
         self.mlog = mlog
+        self.cicmds = cicmds
         self.conftime = conftime
         self.buildtime = buildtime
         self.testtime = testtime
@@ -271,6 +273,33 @@ def yellow(text):
     return mlog.yellow(text).get_text(mlog.colorize_console)
 
 
+def _run_ci_include(args: typing.List[str]) -> str:
+    if not args:
+        return 'At least one parameter required'
+    try:
+        file_path = Path(args[0])
+        data = file_path.open(errors='ignore', encoding='utf-8').read()
+        return 'Included file {}:\n{}\n'.format(args[0], data)
+    except Exception:
+        return 'Failed to open {} ({})'.format(args[0])
+    return 'Appended {} to the log'.format(args[0])
+
+ci_commands = {
+    'ci_include': _run_ci_include
+}
+
+def run_ci_commands(raw_log: str) -> typing.List[str]:
+    res = []
+    for l in raw_log.splitlines():
+        if not l.startswith('!meson_ci!/'):
+            continue
+        cmd = shlex.split(l[11:])
+        if not cmd or cmd[0] not in ci_commands:
+            continue
+        res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))]
+    return res
+
+
 def run_test_inprocess(testdir):
     old_stdout = sys.stdout
     sys.stdout = mystdout = StringIO()
@@ -372,16 +401,17 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
         mesonlog = logfile.open(errors='ignore', encoding='utf-8').read()
     except Exception:
         mesonlog = no_meson_log_msg
+    cicmds = run_ci_commands(mesonlog)
     gen_time = time.time() - gen_start
     if should_fail == 'meson':
         if returncode == 1:
-            return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+            return TestResult('', BuildStep.configure, stdo, stde, mesonlog, cicmds, gen_time)
         elif returncode != 0:
-            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time)
+            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, cicmds, gen_time)
         else:
-            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, cicmds, gen_time)
     if returncode != 0:
-        return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, cicmds, gen_time)
     builddata = build.load(test_build_dir)
     # Touch the meson.build file to force a regenerate so we can test that
     # regeneration works before a build is run.
@@ -396,10 +426,10 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
     stde += e
     if should_fail == 'build':
         if pc.returncode != 0:
-            return TestResult('', BuildStep.build, stdo, stde, mesonlog, gen_time)
-        return TestResult('Test that should have failed to build succeeded', BuildStep.build, stdo, stde, mesonlog, gen_time)
+            return TestResult('', BuildStep.build, stdo, stde, mesonlog, cicmds, gen_time)
+        return TestResult('Test that should have failed to build succeeded', BuildStep.build, stdo, stde, mesonlog, cicmds, gen_time)
     if pc.returncode != 0:
-        return TestResult('Compiling source code failed.', BuildStep.build, stdo, stde, mesonlog, gen_time, build_time)
+        return TestResult('Compiling source code failed.', BuildStep.build, stdo, stde, mesonlog, cicmds, gen_time, build_time)
     # Touch the meson.build file to force a regenerate so we can test that
     # regeneration works after a build is complete.
     ensure_backend_detects_changes(backend)
@@ -413,10 +443,10 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
     mesonlog += test_log
     if should_fail == 'test':
         if returncode != 0:
-            return TestResult('', BuildStep.test, stdo, stde, mesonlog, gen_time)
-        return TestResult('Test that should have failed to run unit tests succeeded', BuildStep.test, stdo, stde, mesonlog, gen_time)
+            return TestResult('', BuildStep.test, stdo, stde, mesonlog, cicmds, gen_time)
+        return TestResult('Test that should have failed to run unit tests succeeded', BuildStep.test, stdo, stde, mesonlog, cicmds, gen_time)
     if returncode != 0:
-        return TestResult('Running unit tests failed.', BuildStep.test, stdo, stde, mesonlog, gen_time, build_time, test_time)
+        return TestResult('Running unit tests failed.', BuildStep.test, stdo, stde, mesonlog, cicmds, gen_time, build_time, test_time)
     # Do installation, if the backend supports it
     if install_commands:
         env = os.environ.copy()
@@ -426,18 +456,18 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
         stdo += o
         stde += e
         if pi.returncode != 0:
-            return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, gen_time, build_time, test_time)
+            return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, cicmds, gen_time, build_time, test_time)
     # Clean with subprocess
     env = os.environ.copy()
     pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env)
     stdo += o
     stde += e
     if pi.returncode != 0:
-        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
+        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, cicmds, gen_time, build_time, test_time)
     if not install_commands:
-        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time, build_time, test_time)
+        return TestResult('', BuildStep.install, '', '', mesonlog, cicmds, gen_time, build_time, test_time)
     return TestResult(validate_install(testdir, install_dir, compiler, builddata.environment),
-                      BuildStep.validate, stdo, stde, mesonlog, gen_time, build_time, test_time)
+                      BuildStep.validate, stdo, stde, mesonlog, cicmds, gen_time, build_time, test_time)
 
 def gather_tests(testdir: Path) -> typing.List[Path]:
     test_names = [t.name for t in testdir.glob('*') if t.is_dir()]
@@ -746,6 +776,8 @@ def _run_tests(all_tests: typing.List[typing.Tuple[str, typing.List[Path], bool]
                         failing_logs.append(result.stdo)
                     else:
                         failing_logs.append(result.stdo)
+                    for cmd_res in result.cicmds:
+                        failing_logs.append(cmd_res)
                     failing_logs.append(result.stde)
                     if failfast:
                         print("Cancelling the rest of the tests")

test cases/cmake/13 system includes/meson.build

@@ -2,10 +2,14 @@ project(
   'meson_cmake_system_include_bug', ['c', 'cpp'],
   default_options: [
     'warning_level=3',
-    #'werror=true', # TODO implement system includes
+    'werror=true',
   ],
 )
 
+if meson.get_compiler('cpp').get_argument_syntax() == 'msvc'
+  error('MESON_SKIP_TEST: Skipp with msvc due to missing -system support')
+endif
+
 cm = import('cmake')
 sub_pro = cm.subproject('cmMod')
 sub_dep = sub_pro.dependency('cmModLib')