Merge pull request #10916 from xclaesse/preprocess

Add cc.preprocess() method

docs/markdown/snippets/preprocess.md

@@ -0,0 +1,14 @@
+## New method to preprocess source files
+
+Compiler object has a new `preprocess()` method. It is supported by all C/C++
+compilers. It preprocess sources without compiling them.
+
+The preprocessor will receive the same arguments (include directories, defines,
+etc) as with normal compilation. That includes for example args added with
+`add_project_arguments()`, or on the command line with `-Dc_args=-DFOO`.
+
+```meson
+cc = meson.get_compiler('c')
+pp_files = cc.preprocess('foo.c', 'bar.c', output: '@PLAINNAME@')
+exe = executable('app', pp_files)
+```

docs/yaml/objects/compiler.yaml

@@ -586,3 +586,25 @@ methods:
     gcc or msvc, but use the same argument syntax as one of those two compilers
     such as clang or icc, especially when they use different syntax on different
     operating systems.
+
+- name: preprocess
+  returns: list[custom_idx]
+  since: 0.64.0
+  description: |
+    Preprocess a list of source files but do not compile them. The preprocessor
+    will receive the same arguments (include directories, defines, etc) as with
+    normal compilation. That includes for example args added with
+    `add_project_arguments()`, or on the command line with `-Dc_args=-DFOO`.
+  varargs_inherit: _build_target_base
+  kwargs:
+    output:
+      type: str
+      description: |
+        Template for name of preprocessed files: `@PLAINNAME@` is replaced by
+        the source filename and `@BASENAME@` is replaced by the source filename
+        without its extension.
+    compile_args:
+      type: list[str]
+      description: |
+        Extra flags to pass to the preprocessor
+

mesonbuild/backend/backends.py

@@ -50,6 +50,8 @@ if T.TYPE_CHECKING:
 
     from typing_extensions import TypedDict
 
+    _ALL_SOURCES_TYPE = T.List[T.Union[File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList]]
+
     class TargetIntrospectionData(TypedDict):
 
         language: str
@@ -792,6 +794,8 @@ class Backend:
 
     def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOrString') -> str:
         assert isinstance(source, mesonlib.File)
+        if isinstance(target, build.CompileTarget):
+            return target.sources_map[source]
         build_dir = self.environment.get_build_dir()
         rel_src = source.rel_to_builddir(self.build_to_src)
 
@@ -1862,3 +1866,28 @@ class Backend:
             else:
                 env.prepend('PATH', list(extra_paths))
         return env
+
+    def compiler_to_generator(self, target: build.BuildTarget,
+                              compiler: 'Compiler',
+                              sources: _ALL_SOURCES_TYPE,
+                              output_templ: str) -> build.GeneratedList:
+        '''
+        Some backends don't support custom compilers. This is a convenience
+        method to convert a Compiler to a Generator.
+        '''
+        exelist = compiler.get_exelist()
+        exe = programs.ExternalProgram(exelist[0])
+        args = exelist[1:]
+        # FIXME: There are many other args missing
+        commands = self.generate_basic_compiler_args(target, compiler)
+        commands += compiler.get_dependency_gen_args('@OUTPUT@', '@DEPFILE@')
+        commands += compiler.get_output_args('@OUTPUT@')
+        commands += compiler.get_compile_only_args() + ['@INPUT@']
+        commands += self.get_source_dir_include_args(target, compiler)
+        commands += self.get_build_dir_include_args(target, compiler)
+        generator = build.Generator(exe, args + commands.to_native(), [output_templ], depfile='@PLAINNAME@.d')
+        return generator.process_files(sources, self.interpreter)
+
+    def compile_target_to_generator(self, target: build.CompileTarget) -> build.GeneratedList:
+        all_sources = T.cast('_ALL_SOURCES_TYPE', target.sources) + T.cast('_ALL_SOURCES_TYPE', target.generated)
+        return self.compiler_to_generator(target, target.compiler, all_sources, target.output_templ)

mesonbuild/backend/ninjabackend.py

@@ -666,10 +666,10 @@ class NinjaBackend(backends.Backend):
         # TODO: Rather than an explicit list here, rules could be marked in the
         # rule store as being wanted in compdb
         for for_machine in MachineChoice:
-            for lang in self.environment.coredata.compilers[for_machine]:
+            for compiler in self.environment.coredata.compilers[for_machine].values():
-                rules += [f"{rule}{ext}" for rule in [self.get_compiler_rule_name(lang, for_machine)]
+                rules += [f"{rule}{ext}" for rule in [self.compiler_to_rule_name(compiler)]
                           for ext in ['', '_RSP']]
-                rules += [f"{rule}{ext}" for rule in [self.get_pch_rule_name(lang, for_machine)]
+                rules += [f"{rule}{ext}" for rule in [self.compiler_to_pch_rule_name(compiler)]
                           for ext in ['', '_RSP']]
         compdb_options = ['-x'] if mesonlib.version_compare(self.ninja_version, '>=1.9') else []
         ninja_compdb = self.ninja_command + ['-t', 'compdb'] + compdb_options + rules
@@ -986,6 +986,9 @@ class NinjaBackend(backends.Backend):
                 obj_list.append(o)
                 compiled_sources.append(s)
                 source2object[s] = o
+        if isinstance(target, build.CompileTarget):
+            # Skip the link stage for this special type of target
+            return
         linker, stdlib_args = self.determine_linker_and_stdlib_args(target)
         if isinstance(target, build.StaticLibrary) and target.prelink:
             final_obj_list = self.generate_prelink(target, obj_list)
@@ -1426,7 +1429,7 @@ class NinjaBackend(backends.Backend):
         commands += self.build.get_project_args(compiler, target.subproject, target.for_machine)
         commands += self.build.get_global_args(compiler, target.for_machine)
 
-        elem = NinjaBuildElement(self.all_outputs, outputs, self.get_compiler_rule_name('cs', target.for_machine), rel_srcs + generated_rel_srcs)
+        elem = NinjaBuildElement(self.all_outputs, outputs, self.compiler_to_rule_name(compiler), rel_srcs + generated_rel_srcs)
         elem.add_dep(deps)
         elem.add_item('ARGS', commands)
         self.add_build(elem)
@@ -1959,7 +1962,7 @@ class NinjaBackend(backends.Backend):
                                      getattr(target, 'rust_crate_type', '') == 'procmacro',
                                      output, project_deps)
 
-        compiler_name = self.get_compiler_rule_name('rust', target.for_machine)
+        compiler_name = self.compiler_to_rule_name(rustc)
         element = NinjaBuildElement(self.all_outputs, target_name, compiler_name, main_rust_file)
         if orderdeps:
             element.add_orderdep(orderdeps)
@@ -1978,20 +1981,16 @@ class NinjaBackend(backends.Backend):
         return PerMachine('_FOR_BUILD', '')[for_machine]
 
     @classmethod
-    def get_compiler_rule_name(cls, lang: str, for_machine: MachineChoice) -> str:
+    def get_compiler_rule_name(cls, lang: str, for_machine: MachineChoice, mode: str = 'COMPILER') -> str:
-        return '{}_COMPILER{}'.format(lang, cls.get_rule_suffix(for_machine))
+        return f'{lang}_{mode}{cls.get_rule_suffix(for_machine)}'
-
-    @classmethod
-    def get_pch_rule_name(cls, lang: str, for_machine: MachineChoice) -> str:
-        return '{}_PCH{}'.format(lang, cls.get_rule_suffix(for_machine))
 
     @classmethod
     def compiler_to_rule_name(cls, compiler: Compiler) -> str:
-        return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine)
+        return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine, compiler.mode)
 
     @classmethod
     def compiler_to_pch_rule_name(cls, compiler: Compiler) -> str:
-        return cls.get_pch_rule_name(compiler.get_language(), compiler.for_machine)
+        return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine, 'PCH')
 
     def swift_module_file_name(self, target):
         return os.path.join(self.get_target_private_dir(target),
@@ -2090,7 +2089,7 @@ class NinjaBackend(backends.Backend):
             objects.append(oname)
             rel_objects.append(os.path.join(self.get_target_private_dir(target), oname))
 
-        rulename = self.get_compiler_rule_name('swift', target.for_machine)
+        rulename = self.compiler_to_rule_name(swiftc)
 
         # Swiftc does not seem to be able to emit objects and module files in one go.
         elem = NinjaBuildElement(self.all_outputs, rel_objects, rulename, abssrc)
@@ -2099,9 +2098,7 @@ class NinjaBackend(backends.Backend):
         elem.add_item('ARGS', compile_args + header_imports + abs_generated + module_includes)
         elem.add_item('RUNDIR', rundir)
         self.add_build(elem)
-        elem = NinjaBuildElement(self.all_outputs, out_module_name,
+        elem = NinjaBuildElement(self.all_outputs, out_module_name, rulename, abssrc)
-                                 self.get_compiler_rule_name('swift', target.for_machine),
-                                 abssrc)
         elem.add_dep(in_module_files + rel_generated)
         elem.add_item('ARGS', compile_args + abs_generated + module_includes + swiftc.get_mod_gen_args())
         elem.add_item('RUNDIR', rundir)
@@ -2312,7 +2309,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         crstr = self.get_rule_suffix(compiler.for_machine)
         if langname == 'fortran':
             self.generate_fortran_dep_hack(crstr)
-        rule = self.get_compiler_rule_name(langname, compiler.for_machine)
+        rule = self.compiler_to_rule_name(compiler)
         depargs = NinjaCommandArg.list(compiler.get_dependency_gen_args('$out', '$DEPFILE'), Quoting.none)
         command = compiler.get_exelist()
         args = ['$ARGS'] + depargs + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
@@ -2368,6 +2365,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
                     self.generate_llvm_ir_compile_rule(compiler)
                 self.generate_compile_rule_for(langname, compiler)
                 self.generate_pch_rule_for(langname, compiler)
+                for mode in compiler.get_modes():
+                    self.generate_compile_rule_for(langname, mode)
 
     def generate_generator_list_rules(self, target):
         # CustomTargets have already written their rules and

mesonbuild/backend/vs2010backend.py

@@ -683,6 +683,23 @@ class Vs2010Backend(backends.Backend):
         self.add_target_deps(root, target)
         self._prettyprint_vcxproj_xml(ET.ElementTree(root), ofname)
 
+    def gen_compile_target_vcxproj(self, target, ofname, guid):
+        if target.for_machine is MachineChoice.BUILD:
+            platform = self.build_platform
+        else:
+            platform = self.platform
+        (root, type_config) = self.create_basic_project(target.name,
+                                                        temp_dir=target.get_id(),
+                                                        guid=guid,
+                                                        target_platform=platform)
+        ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets')
+        target.generated = [self.compile_target_to_generator(target)]
+        target.sources = []
+        self.generate_custom_generator_commands(target, root)
+        self.add_regen_dependency(root)
+        self.add_target_deps(root, target)
+        self._prettyprint_vcxproj_xml(ET.ElementTree(root), ofname)
+
     @classmethod
     def lang_from_source_file(cls, src):
         ext = src.split('.')[-1]
@@ -876,6 +893,8 @@ class Vs2010Backend(backends.Backend):
             return self.gen_custom_target_vcxproj(target, ofname, guid)
         elif isinstance(target, build.RunTarget):
             return self.gen_run_target_vcxproj(target, ofname, guid)
+        elif isinstance(target, build.CompileTarget):
+            return self.gen_compile_target_vcxproj(target, ofname, guid)
         else:
             raise MesonException(f'Unknown target type for {target.get_basename()}')
         # Prefix to use to access the build root from the vcxproj dir

mesonbuild/build.py

@@ -2583,6 +2583,44 @@ class CustomTarget(Target, CommandBase):
     def __len__(self) -> int:
         return len(self.outputs)
 
+class CompileTarget(BuildTarget):
+    '''
+    Target that only compile sources without linking them together.
+    It can be used as preprocessor, or transpiler.
+    '''
+
+    typename = 'compile'
+
+    def __init__(self,
+                 name: str,
+                 subdir: str,
+                 subproject: str,
+                 environment: environment.Environment,
+                 sources: T.List[File],
+                 output_templ: str,
+                 compiler: Compiler,
+                 kwargs):
+        compilers = {compiler.get_language(): compiler}
+        super().__init__(name, subdir, subproject, compiler.for_machine,
+                         sources, None, [], environment, compilers, kwargs)
+        self.filename = name
+        self.compiler = compiler
+        self.output_templ = output_templ
+        self.outputs = []
+        for f in sources:
+            plainname = os.path.basename(f.fname)
+            basename = os.path.splitext(plainname)[0]
+            self.outputs.append(output_templ.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname))
+        self.sources_map = dict(zip(sources, self.outputs))
+
+    def type_suffix(self) -> str:
+        return "@compile"
+
+    @property
+    def is_unity(self) -> bool:
+        return False
+
+
 class RunTarget(Target, CommandBase):
 
     typename = 'run'
@@ -2707,7 +2745,7 @@ class CustomTargetIndex(HoldableObject):
 
     typename: T.ClassVar[str] = 'custom'
 
-    target: CustomTarget
+    target: T.Union[CustomTarget, CompileTarget]
     output: str
 
     def __post_init__(self) -> None:
@@ -2718,8 +2756,7 @@ class CustomTargetIndex(HoldableObject):
         return f'{self.target.name}[{self.output}]'
 
     def __repr__(self):
-        return '<CustomTargetIndex: {!r}[{}]>'.format(
+        return '<CustomTargetIndex: {!r}[{}]>'.format(self.target, self.output)
-            self.target, self.target.get_outputs().index(self.output))
 
     def get_outputs(self) -> T.List[str]:
         return [self.output]

mesonbuild/compilers/compilers.py

@@ -494,6 +494,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
     language: str
     id: str
     warn_args: T.Dict[str, T.List[str]]
+    mode: str = 'COMPILER'
 
     def __init__(self, exelist: T.List[str], version: str,
                  for_machine: MachineChoice, info: 'MachineInfo',
@@ -513,6 +514,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
         self.linker = linker
         self.info = info
         self.is_cross = is_cross
+        self.modes: T.List[Compiler] = []
 
     def __repr__(self) -> str:
         repr_str = "<{0}: v{1} `{2}`>"
@@ -531,6 +533,9 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
     def get_id(self) -> str:
         return self.id
 
+    def get_modes(self) -> T.List[Compiler]:
+        return self.modes
+
     def get_linker_id(self) -> str:
         # There is not guarantee that we have a dynamic linker instance, as
         # some languages don't have separate linkers and compilers. In those
@@ -1050,6 +1055,9 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
     def get_preprocess_only_args(self) -> T.List[str]:
         raise EnvironmentException('This compiler does not have a preprocessor')
 
+    def get_preprocess_to_file_args(self) -> T.List[str]:
+        return self.get_preprocess_only_args()
+
     def get_default_include_dirs(self) -> T.List[str]:
         # TODO: This is a candidate for returning an immutable list
         return []
@@ -1290,6 +1298,10 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
     def needs_static_linker(self) -> bool:
         raise NotImplementedError(f'There is no static linker for {self.language}')
 
+    def get_preprocessor(self) -> Compiler:
+        """Get compiler's preprocessor.
+        """
+        raise EnvironmentException(f'{self.get_id()} does not support preprocessor')
 
 def get_global_options(lang: str,
                        comp: T.Type[Compiler],

mesonbuild/compilers/mixins/clike.py

@@ -27,6 +27,7 @@ import itertools
 import os
 import re
 import subprocess
+import copy
 import typing as T
 from pathlib import Path
 
@@ -145,6 +146,8 @@ class CLikeCompiler(Compiler):
             self.exe_wrapper = None
         else:
             self.exe_wrapper = exe_wrapper
+        # Lazy initialized in get_preprocessor()
+        self.preprocessor: T.Optional[Compiler] = None
 
     def compiler_args(self, args: T.Optional[T.Iterable[str]] = None) -> CLikeCompilerArgs:
         # This is correct, mypy just doesn't understand co-operative inheritance
@@ -1328,3 +1331,18 @@ class CLikeCompiler(Compiler):
 
     def get_disable_assert_args(self) -> T.List[str]:
         return ['-DNDEBUG']
+
+    @functools.lru_cache(maxsize=None)
+    def can_compile(self, src: 'mesonlib.FileOrString') -> bool:
+        # Files we preprocess can be anything, e.g. .in
+        if self.mode == 'PREPROCESSOR':
+            return True
+        return super().can_compile(src)
+
+    def get_preprocessor(self) -> Compiler:
+        if not self.preprocessor:
+            self.preprocessor = copy.copy(self)
+            self.preprocessor.exelist = self.exelist + self.get_preprocess_to_file_args()
+            self.preprocessor.mode = 'PREPROCESSOR'
+            self.modes.append(self.preprocessor)
+        return self.preprocessor

mesonbuild/compilers/mixins/gnu.py

@@ -318,6 +318,12 @@ class GnuLikeCompiler(Compiler, metaclass=abc.ABCMeta):
     def get_coverage_args(self) -> T.List[str]:
         return ['--coverage']
 
+    def get_preprocess_to_file_args(self) -> T.List[str]:
+        # We want to allow preprocessing files with any extension, such as
+        # foo.c.in. In that case we need to tell GCC/CLANG to treat them as
+        # assembly file.
+        return self.get_preprocess_only_args() + ['-x', 'assembler-with-cpp']
+
 
 class GnuCompiler(GnuLikeCompiler):
     """

mesonbuild/compilers/mixins/visualstudio.py

@@ -159,6 +159,9 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta):
     def get_preprocess_only_args(self) -> T.List[str]:
         return ['/EP']
 
+    def get_preprocess_to_file_args(self) -> T.List[str]:
+        return ['/EP', '/P']
+
     def get_compile_only_args(self) -> T.List[str]:
         return ['/c']
 
@@ -173,6 +176,8 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta):
         return ['/fsanitize=address']
 
     def get_output_args(self, target: str) -> T.List[str]:
+        if self.mode == 'PREPROCESSOR':
+            return ['/Fi' + target]
         if target.endswith('.exe'):
             return ['/Fe' + target]
         return ['/Fo' + target]

mesonbuild/interpreter/compiler.py

@@ -4,6 +4,7 @@
 
 import enum
 import functools
+import os
 import typing as T
 
 from .. import build
@@ -79,6 +80,11 @@ if T.TYPE_CHECKING:
         header_prefix: str
         header_required: T.Union[bool, coredata.UserFeatureOption]
 
+    class PreprocessKW(TypedDict):
+        output: str
+        compile_args: T.List[str]
+        include_directories: T.List[build.IncludeDirs]
+
 
 class _TestMode(enum.Enum):
 
@@ -184,6 +190,7 @@ class CompilerHolder(ObjectHolder['Compiler']):
                              'first_supported_link_argument': self.first_supported_link_argument_method,
                              'symbols_have_underscore_prefix': self.symbols_have_underscore_prefix_method,
                              'get_argument_syntax': self.get_argument_syntax_method,
+                             'preprocess': self.preprocess_method,
                              })
 
     @property
@@ -734,3 +741,34 @@ class CompilerHolder(ObjectHolder['Compiler']):
     @noKwargs
     def get_argument_syntax_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> str:
         return self.compiler.get_argument_syntax()
+
+    @FeatureNew('compiler.preprocess', '0.64.0')
+    @typed_pos_args('compiler.preprocess', varargs=(mesonlib.File, str), min_varargs=1)
+    @typed_kwargs(
+        'compiler.preprocess',
+        KwargInfo('output', str, default='@PLAINNAME@.i'),
+        KwargInfo('compile_args', ContainerTypeInfo(list, str), listify=True, default=[]),
+        _INCLUDE_DIRS_KW,
+    )
+    def preprocess_method(self, args: T.Tuple[T.List['mesonlib.FileOrString']], kwargs: 'PreprocessKW') -> T.List[build.CustomTargetIndex]:
+        compiler = self.compiler.get_preprocessor()
+        sources = self.interpreter.source_strings_to_files(args[0])
+        tg_kwargs = {
+            f'{self.compiler.language}_args': kwargs['compile_args'],
+            'build_by_default': False,
+            'include_directories': kwargs['include_directories'],
+        }
+        tg = build.CompileTarget(
+            'preprocessor',
+            self.interpreter.subdir,
+            self.subproject,
+            self.environment,
+            sources,
+            kwargs['output'],
+            compiler,
+            tg_kwargs)
+        self.interpreter.add_target(tg.name, tg)
+        # Expose this target as list of its outputs, so user can pass them to
+        # other targets, list outputs, etc.
+        private_dir = os.path.relpath(self.interpreter.backend.get_target_private_dir(tg), self.interpreter.subdir)
+        return [build.CustomTargetIndex(tg, os.path.join(private_dir, o)) for o in tg.outputs]

test cases/common/255 preprocess/bar.c

@@ -0,0 +1,3 @@
+int bar(void) {
+    return BAR;
+}

test cases/common/255 preprocess/foo.c

@@ -0,0 +1 @@
+#include <foo.h>

test cases/common/255 preprocess/foo.h

@@ -0,0 +1,2 @@
+int bar(void);
+int main(void) { return FOO + bar(); }

test cases/common/255 preprocess/meson.build

@@ -0,0 +1,15 @@
+project('preprocess', 'c')
+
+cc = meson.get_compiler('c')
+
+add_project_arguments(['-DFOO=0', '-DBAR=0'], language: 'c')
+
+pp_files = cc.preprocess('foo.c', 'bar.c', output: '@PLAINNAME@')
+
+foreach f : pp_files
+  message(f.full_path())
+endforeach
+
+subdir('src')
+
+test('test-foo', executable('app', pp_files, link_depends: file_map))

test cases/common/255 preprocess/src/file.map.in

@@ -0,0 +1,3 @@
+#if 1
+Hello World
+#endif

test cases/common/255 preprocess/src/meson.build

@@ -0,0 +1,3 @@
+file_map = cc.preprocess('file.map.in',
+  output: '@BASENAME@',
+)