Merge pull request #6432 from mensinda/cmExeRefactor

cmake: Refactor CMakeExecutor and CMakeTraceParser

mesonbuild/cmake/executor.py

@@ -15,8 +15,9 @@
 # This class contains the basic functionality needed to run any interpreter
 # or an interpreter-based tool.
 
-import subprocess
+import subprocess as S
 from pathlib import Path
+from threading import Thread
 import typing as T
 import re
 import os
@@ -30,19 +31,22 @@ from ..environment import Environment
 if T.TYPE_CHECKING:
     from ..dependencies.base import ExternalProgram
 
+TYPE_result = T.Tuple[int, T.Optional[str], T.Optional[str]]
 
 class CMakeExecutor:
     # The class's copy of the CMake path. Avoids having to search for it
     # multiple times in the same Meson invocation.
     class_cmakebin = PerMachine(None, None)
     class_cmakevers = PerMachine(None, None)
-    class_cmake_cache = {}
+    class_cmake_cache = {}  # type: T.Dict[T.Any, TYPE_result]
 
     def __init__(self, environment: Environment, version: str, for_machine: MachineChoice, silent: bool = False):
         self.min_version = version
         self.environment = environment
         self.for_machine = for_machine
         self.cmakebin, self.cmakevers = self.find_cmake_binary(self.environment, silent=silent)
+        self.always_capture_stderr = True
+        self.print_cmout = False
         if self.cmakebin is False:
             self.cmakebin = None
             return
@@ -130,17 +134,77 @@ class CMakeExecutor:
         cmvers = re.sub(r'\s*cmake version\s*', '', out.split('\n')[0]).strip()
         return cmvers
 
+    def set_exec_mode(self, print_cmout: T.Optional[bool] = None, always_capture_stderr: T.Optional[bool] = None) -> None:
+        if print_cmout is not None:
+            self.print_cmout = print_cmout
+        if always_capture_stderr is not None:
+            self.always_capture_stderr = always_capture_stderr
+
     def _cache_key(self, args: T.List[str], build_dir: str, env):
         fenv = frozenset(env.items()) if env is not None else None
         targs = tuple(args)
         return (self.cmakebin, targs, build_dir, fenv)
 
-    def _call_real(self, args: T.List[str], build_dir: str, env) -> T.Tuple[int, str, str]:
+    def _call_cmout_stderr(self, args: T.List[str], build_dir: str, env) -> TYPE_result:
+        cmd = self.cmakebin.get_command() + args
+        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.PIPE, cwd=build_dir, env=env)
+
+        # stdout and stderr MUST be read at the same time to avoid pipe
+        # blocking issues. The easiest way to do this is with a separate
+        # thread for one of the pipes.
+        def print_stdout():
+            while True:
+                line = proc.stdout.readline()
+                if not line:
+                    break
+                mlog.log(line.decode(errors='ignore').strip('\n'))
+            proc.stdout.close()
+
+        t = Thread(target=print_stdout)
+        t.start()
+
+        try:
+            # Read stderr line by line and log non trace lines
+            raw_trace = ''
+            tline_start_reg = re.compile(r'^\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(.*$')
+            inside_multiline_trace = False
+            while True:
+                line = proc.stderr.readline()
+                if not line:
+                    break
+                line = line.decode(errors='ignore')
+                if tline_start_reg.match(line):
+                    raw_trace += line
+                    inside_multiline_trace = not line.endswith(' )\n')
+                elif inside_multiline_trace:
+                    raw_trace += line
+                else:
+                    mlog.warning(line.strip('\n'))
+
+        finally:
+            proc.stderr.close()
+            t.join()
+            proc.wait()
+
+        return proc.returncode, None, raw_trace
+
+    def _call_cmout(self, args: T.List[str], build_dir: str, env) -> TYPE_result:
+        cmd = self.cmakebin.get_command() + args
+        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.STDOUT, cwd=build_dir, env=env)
+        while True:
+            line = proc.stdout.readline()
+            if not line:
+                break
+            mlog.log(line.decode(errors='ignore').strip('\n'))
+        proc.stdout.close()
+        proc.wait()
+        return proc.returncode, None, None
+
+    def _call_quiet(self, args: T.List[str], build_dir: str, env) -> TYPE_result:
         os.makedirs(build_dir, exist_ok=True)
         cmd = self.cmakebin.get_command() + args
-        ret = subprocess.run(cmd, env=env, cwd=build_dir, close_fds=False,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             universal_newlines=False)
+        ret = S.run(cmd, env=env, cwd=build_dir, close_fds=False,
+                    stdout=S.PIPE, stderr=S.PIPE, universal_newlines=False)
         rc = ret.returncode
         out = ret.stdout.decode(errors='ignore')
         err = ret.stderr.decode(errors='ignore')
@@ -148,21 +212,30 @@ class CMakeExecutor:
         mlog.debug("Called `{}` in {} -> {}".format(call, build_dir, rc))
         return rc, out, err
 
-    def call(self, args: T.List[str], build_dir: str, env=None, disable_cache: bool = False):
+    def _call_impl(self, args: T.List[str], build_dir: str, env) -> TYPE_result:
+        if not self.print_cmout:
+            return self._call_quiet(args, build_dir, env)
+        else:
+            if self.always_capture_stderr:
+                return self._call_cmout_stderr(args, build_dir, env)
+            else:
+                return self._call_cmout(args, build_dir, env)
+
+    def call(self, args: T.List[str], build_dir: str, env=None, disable_cache: bool = False) -> TYPE_result:
         if env is None:
             env = os.environ
 
         if disable_cache:
-            return self._call_real(args, build_dir, env)
+            return self._call_impl(args, build_dir, env)
 
         # First check if cached, if not call the real cmake function
         cache = CMakeExecutor.class_cmake_cache
         key = self._cache_key(args, build_dir, env)
         if key not in cache:
-            cache[key] = self._call_real(args, build_dir, env)
+            cache[key] = self._call_impl(args, build_dir, env)
         return cache[key]
 
-    def call_with_fake_build(self, args: T.List[str], build_dir: str, env=None):
+    def call_with_fake_build(self, args: T.List[str], build_dir: str, env=None) -> TYPE_result:
         # First check the cache
         cache = CMakeExecutor.class_cmake_cache
         key = self._cache_key(args, build_dir, env)
@@ -282,7 +355,7 @@ set(CMAKE_SIZEOF_VOID_P "{}")
     def executable_path(self) -> str:
         return self.cmakebin.get_path()
 
-    def get_command(self):
+    def get_command(self) -> T.List[str]:
         return self.cmakebin.get_command()
 
     def machine_choice(self) -> MachineChoice:

mesonbuild/cmake/interpreter.py

@@ -24,8 +24,6 @@ from .. import mlog
 from ..environment import Environment
 from ..mesonlib import MachineChoice, version_compare
 from ..compilers.compilers import lang_suffixes, header_suffixes, obj_suffixes, lib_suffixes, is_header
-from subprocess import Popen, PIPE
-from threading import Thread
 from enum import Enum
 from functools import lru_cache
 import typing as T
@@ -741,7 +739,7 @@ class CMakeInterpreter:
         self.languages = []
         self.targets = []
         self.custom_targets = []  # type: T.List[ConverterCustomTarget]
-        self.trace = CMakeTraceParser()
+        self.trace = CMakeTraceParser('', '')  # Will be replaced in analyse
         self.output_target_map = OutputTargetMap(self.build_dir)
 
         # Generated meson data
@@ -754,10 +752,11 @@ class CMakeInterpreter:
         cmake_exe = CMakeExecutor(self.env, '>=3.7', for_machine)
         if not cmake_exe.found():
             raise CMakeException('Unable to find CMake')
+        self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True)
 
         generator = backend_generator_map[self.backend_name]
-        cmake_args = cmake_exe.get_command()
-        trace_args = ['--trace', '--trace-expand', '--no-warn-unused-cli']
+        cmake_args = []
+        trace_args = self.trace.trace_args()
         cmcmp_args = ['-DCMAKE_POLICY_WARNING_{}=OFF'.format(x) for x in disable_policy_warnings]
 
         if version_compare(cmake_exe.version(), '>=3.14'):
@@ -795,46 +794,15 @@ class CMakeInterpreter:
             os.makedirs(self.build_dir, exist_ok=True)
             os_env = os.environ.copy()
             os_env['LC_ALL'] = 'C'
-            final_command = cmake_args + trace_args + cmcmp_args + [self.src_dir]
-            proc = Popen(final_command, stdout=PIPE, stderr=PIPE, cwd=self.build_dir, env=os_env)
-
-            def print_stdout():
-                while True:
-                    line = proc.stdout.readline()
-                    if not line:
-                        break
-                    mlog.log(line.decode('utf-8').strip('\n'))
-                proc.stdout.close()
-
-            t = Thread(target=print_stdout)
-            t.start()
-
-            # Read stderr line by line and log non trace lines
-            self.raw_trace = ''
-            tline_start_reg = re.compile(r'^\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(.*$')
-            inside_multiline_trace = False
-            while True:
-                line = proc.stderr.readline()
-                if not line:
-                    break
-                line = line.decode('utf-8')
-                if tline_start_reg.match(line):
-                    self.raw_trace += line
-                    inside_multiline_trace = not line.endswith(' )\n')
-                elif inside_multiline_trace:
-                    self.raw_trace += line
-                else:
-                    mlog.warning(line.strip('\n'))
-
-            proc.stderr.close()
-            proc.wait()
+            final_args = cmake_args + trace_args + cmcmp_args + [self.src_dir]
 
-            t.join()
+            cmake_exe.set_exec_mode(print_cmout=True, always_capture_stderr=self.trace.requires_stderr())
+            rc, _, self.raw_trace = cmake_exe.call(final_args, self.build_dir, env=os_env, disable_cache=True)
 
         mlog.log()
-        h = mlog.green('SUCCEEDED') if proc.returncode == 0 else mlog.red('FAILED')
+        h = mlog.green('SUCCEEDED') if rc == 0 else mlog.red('FAILED')
         mlog.log('CMake configuration:', h)
-        if proc.returncode != 0:
+        if rc != 0:
             raise CMakeException('Failed to configure the CMake subproject')
 
     def initialise(self, extra_cmake_options: T.List[str]) -> None:
@@ -889,7 +857,6 @@ class CMakeInterpreter:
         self.languages = []
         self.targets = []
         self.custom_targets = []
-        self.trace = CMakeTraceParser(permissive=True)
 
         # Parse the trace
         self.trace.parse(self.raw_trace)

mesonbuild/cmake/traceparser.py

@@ -18,8 +18,10 @@
 from .common import CMakeException
 from .generator import parse_generator_expressions
 from .. import mlog
+from ..mesonlib import version_compare
 
 import typing as T
+from pathlib import Path
 import re
 import os
 
@@ -60,7 +62,7 @@ class CMakeGeneratorTarget(CMakeTarget):
         self.working_dir = None  # type: T.Optional[str]
 
 class CMakeTraceParser:
-    def __init__(self, permissive: bool = False):
+    def __init__(self, cmake_version: str, build_dir: str, permissive: bool = False):
         # Dict of CMake variables: '<var_name>': ['list', 'of', 'values']
         self.vars = {}
 
@@ -71,10 +73,40 @@ class CMakeTraceParser:
         self.custom_targets = []  # type: T.List[CMakeGeneratorTarget]
 
         self.permissive = permissive  # type: bool
+        self.cmake_version = cmake_version  # type: str
+        self.trace_file = 'cmake_trace.txt'
+        self.trace_file_path = Path(build_dir) / self.trace_file
+        self.trace_format = 'human'
+
+    def trace_args(self) -> T.List[str]:
+        arg_map = {
+            'human': ['--trace', '--trace-expand'],
+        }
+
+        base_args = ['--no-warn-unused-cli']
+        if not self.requires_stderr():
+            base_args += ['--trace-redirect={}'.format(self.trace_file)]
+
+        return arg_map[self.trace_format] + base_args
 
-    def parse(self, trace: str) -> None:
-        # First parse the trace
-        lexer1 = self._lex_trace(trace)
+    def requires_stderr(self) -> bool:
+        return version_compare(self.cmake_version, '<3.16')
+
+    def parse(self, trace: T.Optional[str] = None) -> None:
+        # First load the trace (if required)
+        if not self.requires_stderr():
+            if not self.trace_file_path.exists and not self.trace_file_path.is_file():
+                raise CMakeException('CMake: Trace file "{}" not found'.format(str(self.trace_file_path)))
+            trace = self.trace_file_path.read_text()
+        if not trace:
+            raise CMakeException('CMake: The CMake trace was not provided or is empty')
+
+        # Second parse the trace
+        lexer1 = None
+        if self.trace_format == 'human':
+            lexer1 = self._lex_trace_human(trace)
+        else:
+            raise CMakeException('CMake: Internal error: Invalid trace format {}. Expected [human]'.format(self.trace_format))
 
         # All supported functions
         functions = {
@@ -481,7 +513,7 @@ class CMakeTraceParser:
 
             self.targets[target].properties[i[0]] += i[1]
 
-    def _lex_trace(self, trace):
+    def _lex_trace_human(self, trace):
         # The trace format is: '<file>(<line>):  <func>(<args -- can contain \n> )\n'
         reg_tline = re.compile(r'\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(([\s\S]*?) ?\)\s*\n', re.MULTILINE)
         reg_other = re.compile(r'[^\n]*\n')
@@ -510,7 +542,7 @@ class CMakeTraceParser:
             yield CMakeTraceLine(file, line, func, args)
 
     def _guess_files(self, broken_list: T.List[str]) -> T.List[str]:
-        #Try joining file paths that contain spaces
+        # Try joining file paths that contain spaces
 
         reg_start = re.compile(r'^([A-Za-z]:)?/.*/[^./]+$')
         reg_end = re.compile(r'^.*\.[a-zA-Z]+$')

mesonbuild/dependencies/base.py

@@ -1104,7 +1104,6 @@ class CMakeDependency(ExternalDependency):
         # stored in the pickled coredata and recovered.
         self.cmakebin = None
         self.cmakeinfo = None
-        self.traceparser = CMakeTraceParser()
 
         # Where all CMake "build dirs" are located
         self.cmake_root_dir = environment.scratch_dir
@@ -1112,6 +1111,10 @@ class CMakeDependency(ExternalDependency):
         # T.List of successfully found modules
         self.found_modules = []
 
+        # Initialize with None before the first return to avoid
+        # AttributeError exceptions in derived classes
+        self.traceparser = None  # type: CMakeTraceParser
+
         self.cmakebin = CMakeExecutor(environment, CMakeDependency.class_cmake_version, self.for_machine, silent=self.silent)
         if not self.cmakebin.found():
             self.cmakebin = None
@@ -1121,6 +1124,9 @@ class CMakeDependency(ExternalDependency):
             mlog.debug(msg)
             return
 
+        # Setup the trace parser
+        self.traceparser = CMakeTraceParser(self.cmakebin.version(), self._get_build_dir())
+
         if CMakeDependency.class_cmakeinfo[self.for_machine] is None:
             CMakeDependency.class_cmakeinfo[self.for_machine] = self._get_cmake_info()
         self.cmakeinfo = CMakeDependency.class_cmakeinfo[self.for_machine]
@@ -1166,11 +1172,13 @@ class CMakeDependency(ExternalDependency):
             gen_list += [CMakeDependency.class_working_generator]
         gen_list += CMakeDependency.class_cmake_generators
 
+        temp_parser = CMakeTraceParser(self.cmakebin.version(), self._get_build_dir())
+
         for i in gen_list:
             mlog.debug('Try CMake generator: {}'.format(i if len(i) > 0 else 'auto'))
 
             # Prepare options
-            cmake_opts = ['--trace-expand', '.']
+            cmake_opts = temp_parser.trace_args() + ['.']
             if len(i) > 0:
                 cmake_opts = ['-G', i] + cmake_opts
 
@@ -1190,7 +1198,6 @@ class CMakeDependency(ExternalDependency):
             return None
 
         try:
-            temp_parser = CMakeTraceParser()
             temp_parser.parse(err1)
         except MesonException:
             return None
@@ -1343,7 +1350,8 @@ class CMakeDependency(ExternalDependency):
             mlog.debug('Try CMake generator: {}'.format(i if len(i) > 0 else 'auto'))
 
             # Prepare options
-            cmake_opts = ['--trace-expand', '-DNAME={}'.format(name), '-DARCHS={}'.format(';'.join(self.cmakeinfo['archs']))] + args + ['.']
+            cmake_opts = ['-DNAME={}'.format(name), '-DARCHS={}'.format(';'.join(self.cmakeinfo['archs']))] + args + ['.']
+            cmake_opts += self.traceparser.trace_args()
             cmake_opts += self._extra_cmake_opts()
             if len(i) > 0:
                 cmake_opts = ['-G', i] + cmake_opts
@@ -1514,10 +1522,14 @@ class CMakeDependency(ExternalDependency):
         self.compile_args = compileOptions + compileDefinitions + ['-I{}'.format(x) for x in incDirs]
         self.link_args = libraries
 
-    def _setup_cmake_dir(self, cmake_file: str) -> str:
-        # Setup the CMake build environment and return the "build" directory
+    def _get_build_dir(self) -> str:
         build_dir = Path(self.cmake_root_dir) / 'cmake_{}'.format(self.name)
         build_dir.mkdir(parents=True, exist_ok=True)
+        return str(build_dir)
+
+    def _setup_cmake_dir(self, cmake_file: str) -> str:
+        # Setup the CMake build environment and return the "build" directory
+        build_dir = self._get_build_dir()
 
         # Insert language parameters into the CMakeLists.txt and write new CMakeLists.txt
         src_cmake = Path(__file__).parent / 'data' / cmake_file
@@ -1540,11 +1552,11 @@ cmake_minimum_required(VERSION ${{CMAKE_VERSION}})
 project(MesonTemp LANGUAGES {})
 """.format(' '.join(cmake_language)) + cmake_txt
 
-        cm_file = build_dir / 'CMakeLists.txt'
+        cm_file = Path(build_dir) / 'CMakeLists.txt'
         cm_file.write_text(cmake_txt)
         mlog.cmd_ci_include(cm_file.absolute().as_posix())
 
-        return str(build_dir)
+        return build_dir
 
     def _call_cmake(self, args, cmake_file: str, env=None):
         build_dir = self._setup_cmake_dir(cmake_file)
@@ -1568,7 +1580,7 @@ project(MesonTemp LANGUAGES {})
                      configtool: T.Optional[str] = None, internal: T.Optional[str] = None,
                      default_value: T.Optional[str] = None,
                      pkgconfig_define: T.Optional[T.List[str]] = None) -> T.Union[str, T.List[str]]:
-        if cmake:
+        if cmake and self.traceparser is not None:
             try:
                 v = self.traceparser.vars[cmake]
             except KeyError:

mesonbuild/dependencies/dev.py

@@ -396,6 +396,9 @@ class LLVMDependencyCMake(CMakeDependency):
         self.llvm_opt_modules = stringlistify(extract_as_list(kwargs, 'optional_modules'))
         super().__init__(name='LLVM', environment=env, language='cpp', kwargs=kwargs)
 
+        if self.traceparser is None:
+            return
+
         # Extract extra include directories and definitions
         inc_dirs = self.traceparser.get_cmake_var('PACKAGE_INCLUDE_DIRS')
         defs = self.traceparser.get_cmake_var('PACKAGE_DEFINITIONS')

run_project_tests.py

@@ -856,6 +856,8 @@ def check_format():
             continue
         if 'meson-logs' in root or 'meson-private' in root:
             continue
+        if '__CMake_build' in root:
+            continue
         if '.eggs' in root or '_cache' in root:  # e.g. .mypy_cache
             continue
         for fname in filenames:
@@ -919,7 +921,7 @@ def print_tool_versions():
         {
             'tool': 'cmake',
             'args': ['--version'],
-            'regex': re.compile(r'^cmake version ([0-9]+(\.[0-9]+)*)$'),
+            'regex': re.compile(r'^cmake version ([0-9]+(\.[0-9]+)*(-[a-z0-9]+)?)$'),
             'match_group': 1,
         },
     ]