diff --git a/docs/markdown/snippets/preprocess.md b/docs/markdown/snippets/preprocess.md new file mode 100644 index 000000000..7e6dd0af3 --- /dev/null +++ b/docs/markdown/snippets/preprocess.md @@ -0,0 +1,14 @@ +## New method to preprocess source files + +Compiler object has a new `preprocess()` method. It is supported by all C/C++ +compilers. It preprocess sources without compiling them. + +The preprocessor will receive the same arguments (include directories, defines, +etc) as with normal compilation. That includes for example args added with +`add_project_arguments()`, or on the command line with `-Dc_args=-DFOO`. + +```meson +cc = meson.get_compiler('c') +pp_files = cc.preprocess('foo.c', 'bar.c', output: '@PLAINNAME@') +exe = executable('app', pp_files) +``` diff --git a/docs/yaml/objects/compiler.yaml b/docs/yaml/objects/compiler.yaml index cf341119d..e10e8fe2b 100644 --- a/docs/yaml/objects/compiler.yaml +++ b/docs/yaml/objects/compiler.yaml @@ -586,3 +586,25 @@ methods: gcc or msvc, but use the same argument syntax as one of those two compilers such as clang or icc, especially when they use different syntax on different operating systems. + +- name: preprocess + returns: list[custom_idx] + since: 0.64.0 + description: | + Preprocess a list of source files but do not compile them. The preprocessor + will receive the same arguments (include directories, defines, etc) as with + normal compilation. That includes for example args added with + `add_project_arguments()`, or on the command line with `-Dc_args=-DFOO`. + varargs_inherit: _build_target_base + kwargs: + output: + type: str + description: | + Template for name of preprocessed files: `@PLAINNAME@` is replaced by + the source filename and `@BASENAME@` is replaced by the source filename + without its extension. + compile_args: + type: list[str] + description: | + Extra flags to pass to the preprocessor + diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 8943464cc..32f24e972 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -50,6 +50,8 @@ if T.TYPE_CHECKING: from typing_extensions import TypedDict + _ALL_SOURCES_TYPE = T.List[T.Union[File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList]] + class TargetIntrospectionData(TypedDict): language: str @@ -792,6 +794,8 @@ class Backend: def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOrString') -> str: assert isinstance(source, mesonlib.File) + if isinstance(target, build.CompileTarget): + return target.sources_map[source] build_dir = self.environment.get_build_dir() rel_src = source.rel_to_builddir(self.build_to_src) @@ -1862,3 +1866,28 @@ class Backend: else: env.prepend('PATH', list(extra_paths)) return env + + def compiler_to_generator(self, target: build.BuildTarget, + compiler: 'Compiler', + sources: _ALL_SOURCES_TYPE, + output_templ: str) -> build.GeneratedList: + ''' + Some backends don't support custom compilers. This is a convenience + method to convert a Compiler to a Generator. + ''' + exelist = compiler.get_exelist() + exe = programs.ExternalProgram(exelist[0]) + args = exelist[1:] + # FIXME: There are many other args missing + commands = self.generate_basic_compiler_args(target, compiler) + commands += compiler.get_dependency_gen_args('@OUTPUT@', '@DEPFILE@') + commands += compiler.get_output_args('@OUTPUT@') + commands += compiler.get_compile_only_args() + ['@INPUT@'] + commands += self.get_source_dir_include_args(target, compiler) + commands += self.get_build_dir_include_args(target, compiler) + generator = build.Generator(exe, args + commands.to_native(), [output_templ], depfile='@PLAINNAME@.d') + return generator.process_files(sources, self.interpreter) + + def compile_target_to_generator(self, target: build.CompileTarget) -> build.GeneratedList: + all_sources = T.cast('_ALL_SOURCES_TYPE', target.sources) + T.cast('_ALL_SOURCES_TYPE', target.generated) + return self.compiler_to_generator(target, target.compiler, all_sources, target.output_templ) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 4f2981c25..7d90ac1f4 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -666,10 +666,10 @@ class NinjaBackend(backends.Backend): # TODO: Rather than an explicit list here, rules could be marked in the # rule store as being wanted in compdb for for_machine in MachineChoice: - for lang in self.environment.coredata.compilers[for_machine]: - rules += [f"{rule}{ext}" for rule in [self.get_compiler_rule_name(lang, for_machine)] + for compiler in self.environment.coredata.compilers[for_machine].values(): + rules += [f"{rule}{ext}" for rule in [self.compiler_to_rule_name(compiler)] for ext in ['', '_RSP']] - rules += [f"{rule}{ext}" for rule in [self.get_pch_rule_name(lang, for_machine)] + rules += [f"{rule}{ext}" for rule in [self.compiler_to_pch_rule_name(compiler)] for ext in ['', '_RSP']] compdb_options = ['-x'] if mesonlib.version_compare(self.ninja_version, '>=1.9') else [] ninja_compdb = self.ninja_command + ['-t', 'compdb'] + compdb_options + rules @@ -986,6 +986,9 @@ class NinjaBackend(backends.Backend): obj_list.append(o) compiled_sources.append(s) source2object[s] = o + if isinstance(target, build.CompileTarget): + # Skip the link stage for this special type of target + return linker, stdlib_args = self.determine_linker_and_stdlib_args(target) if isinstance(target, build.StaticLibrary) and target.prelink: final_obj_list = self.generate_prelink(target, obj_list) @@ -1426,7 +1429,7 @@ class NinjaBackend(backends.Backend): commands += self.build.get_project_args(compiler, target.subproject, target.for_machine) commands += self.build.get_global_args(compiler, target.for_machine) - elem = NinjaBuildElement(self.all_outputs, outputs, self.get_compiler_rule_name('cs', target.for_machine), rel_srcs + generated_rel_srcs) + elem = NinjaBuildElement(self.all_outputs, outputs, self.compiler_to_rule_name(compiler), rel_srcs + generated_rel_srcs) elem.add_dep(deps) elem.add_item('ARGS', commands) self.add_build(elem) @@ -1959,7 +1962,7 @@ class NinjaBackend(backends.Backend): getattr(target, 'rust_crate_type', '') == 'procmacro', output, project_deps) - compiler_name = self.get_compiler_rule_name('rust', target.for_machine) + compiler_name = self.compiler_to_rule_name(rustc) element = NinjaBuildElement(self.all_outputs, target_name, compiler_name, main_rust_file) if orderdeps: element.add_orderdep(orderdeps) @@ -1978,20 +1981,16 @@ class NinjaBackend(backends.Backend): return PerMachine('_FOR_BUILD', '')[for_machine] @classmethod - def get_compiler_rule_name(cls, lang: str, for_machine: MachineChoice) -> str: - return '{}_COMPILER{}'.format(lang, cls.get_rule_suffix(for_machine)) - - @classmethod - def get_pch_rule_name(cls, lang: str, for_machine: MachineChoice) -> str: - return '{}_PCH{}'.format(lang, cls.get_rule_suffix(for_machine)) + def get_compiler_rule_name(cls, lang: str, for_machine: MachineChoice, mode: str = 'COMPILER') -> str: + return f'{lang}_{mode}{cls.get_rule_suffix(for_machine)}' @classmethod def compiler_to_rule_name(cls, compiler: Compiler) -> str: - return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine) + return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine, compiler.mode) @classmethod def compiler_to_pch_rule_name(cls, compiler: Compiler) -> str: - return cls.get_pch_rule_name(compiler.get_language(), compiler.for_machine) + return cls.get_compiler_rule_name(compiler.get_language(), compiler.for_machine, 'PCH') def swift_module_file_name(self, target): return os.path.join(self.get_target_private_dir(target), @@ -2090,7 +2089,7 @@ class NinjaBackend(backends.Backend): objects.append(oname) rel_objects.append(os.path.join(self.get_target_private_dir(target), oname)) - rulename = self.get_compiler_rule_name('swift', target.for_machine) + rulename = self.compiler_to_rule_name(swiftc) # Swiftc does not seem to be able to emit objects and module files in one go. elem = NinjaBuildElement(self.all_outputs, rel_objects, rulename, abssrc) @@ -2099,9 +2098,7 @@ class NinjaBackend(backends.Backend): elem.add_item('ARGS', compile_args + header_imports + abs_generated + module_includes) elem.add_item('RUNDIR', rundir) self.add_build(elem) - elem = NinjaBuildElement(self.all_outputs, out_module_name, - self.get_compiler_rule_name('swift', target.for_machine), - abssrc) + elem = NinjaBuildElement(self.all_outputs, out_module_name, rulename, abssrc) elem.add_dep(in_module_files + rel_generated) elem.add_item('ARGS', compile_args + abs_generated + module_includes + swiftc.get_mod_gen_args()) elem.add_item('RUNDIR', rundir) @@ -2312,7 +2309,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) crstr = self.get_rule_suffix(compiler.for_machine) if langname == 'fortran': self.generate_fortran_dep_hack(crstr) - rule = self.get_compiler_rule_name(langname, compiler.for_machine) + rule = self.compiler_to_rule_name(compiler) depargs = NinjaCommandArg.list(compiler.get_dependency_gen_args('$out', '$DEPFILE'), Quoting.none) command = compiler.get_exelist() args = ['$ARGS'] + depargs + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in'] @@ -2368,6 +2365,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) self.generate_llvm_ir_compile_rule(compiler) self.generate_compile_rule_for(langname, compiler) self.generate_pch_rule_for(langname, compiler) + for mode in compiler.get_modes(): + self.generate_compile_rule_for(langname, mode) def generate_generator_list_rules(self, target): # CustomTargets have already written their rules and diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index 98d69e768..f81e8781b 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -683,6 +683,23 @@ class Vs2010Backend(backends.Backend): self.add_target_deps(root, target) self._prettyprint_vcxproj_xml(ET.ElementTree(root), ofname) + def gen_compile_target_vcxproj(self, target, ofname, guid): + if target.for_machine is MachineChoice.BUILD: + platform = self.build_platform + else: + platform = self.platform + (root, type_config) = self.create_basic_project(target.name, + temp_dir=target.get_id(), + guid=guid, + target_platform=platform) + ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets') + target.generated = [self.compile_target_to_generator(target)] + target.sources = [] + self.generate_custom_generator_commands(target, root) + self.add_regen_dependency(root) + self.add_target_deps(root, target) + self._prettyprint_vcxproj_xml(ET.ElementTree(root), ofname) + @classmethod def lang_from_source_file(cls, src): ext = src.split('.')[-1] @@ -876,6 +893,8 @@ class Vs2010Backend(backends.Backend): return self.gen_custom_target_vcxproj(target, ofname, guid) elif isinstance(target, build.RunTarget): return self.gen_run_target_vcxproj(target, ofname, guid) + elif isinstance(target, build.CompileTarget): + return self.gen_compile_target_vcxproj(target, ofname, guid) else: raise MesonException(f'Unknown target type for {target.get_basename()}') # Prefix to use to access the build root from the vcxproj dir diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 1c8013ce5..9f4fe6ae3 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -2583,6 +2583,44 @@ class CustomTarget(Target, CommandBase): def __len__(self) -> int: return len(self.outputs) +class CompileTarget(BuildTarget): + ''' + Target that only compile sources without linking them together. + It can be used as preprocessor, or transpiler. + ''' + + typename = 'compile' + + def __init__(self, + name: str, + subdir: str, + subproject: str, + environment: environment.Environment, + sources: T.List[File], + output_templ: str, + compiler: Compiler, + kwargs): + compilers = {compiler.get_language(): compiler} + super().__init__(name, subdir, subproject, compiler.for_machine, + sources, None, [], environment, compilers, kwargs) + self.filename = name + self.compiler = compiler + self.output_templ = output_templ + self.outputs = [] + for f in sources: + plainname = os.path.basename(f.fname) + basename = os.path.splitext(plainname)[0] + self.outputs.append(output_templ.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname)) + self.sources_map = dict(zip(sources, self.outputs)) + + def type_suffix(self) -> str: + return "@compile" + + @property + def is_unity(self) -> bool: + return False + + class RunTarget(Target, CommandBase): typename = 'run' @@ -2707,7 +2745,7 @@ class CustomTargetIndex(HoldableObject): typename: T.ClassVar[str] = 'custom' - target: CustomTarget + target: T.Union[CustomTarget, CompileTarget] output: str def __post_init__(self) -> None: @@ -2718,8 +2756,7 @@ class CustomTargetIndex(HoldableObject): return f'{self.target.name}[{self.output}]' def __repr__(self): - return ''.format( - self.target, self.target.get_outputs().index(self.output)) + return ''.format(self.target, self.output) def get_outputs(self) -> T.List[str]: return [self.output] diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 53b430787..b3191a844 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -494,6 +494,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): language: str id: str warn_args: T.Dict[str, T.List[str]] + mode: str = 'COMPILER' def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, info: 'MachineInfo', @@ -513,6 +514,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): self.linker = linker self.info = info self.is_cross = is_cross + self.modes: T.List[Compiler] = [] def __repr__(self) -> str: repr_str = "<{0}: v{1} `{2}`>" @@ -531,6 +533,9 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): def get_id(self) -> str: return self.id + def get_modes(self) -> T.List[Compiler]: + return self.modes + def get_linker_id(self) -> str: # There is not guarantee that we have a dynamic linker instance, as # some languages don't have separate linkers and compilers. In those @@ -1050,6 +1055,9 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): def get_preprocess_only_args(self) -> T.List[str]: raise EnvironmentException('This compiler does not have a preprocessor') + def get_preprocess_to_file_args(self) -> T.List[str]: + return self.get_preprocess_only_args() + def get_default_include_dirs(self) -> T.List[str]: # TODO: This is a candidate for returning an immutable list return [] @@ -1290,6 +1298,10 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): def needs_static_linker(self) -> bool: raise NotImplementedError(f'There is no static linker for {self.language}') + def get_preprocessor(self) -> Compiler: + """Get compiler's preprocessor. + """ + raise EnvironmentException(f'{self.get_id()} does not support preprocessor') def get_global_options(lang: str, comp: T.Type[Compiler], diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py index cc78639e6..e1baa849b 100644 --- a/mesonbuild/compilers/mixins/clike.py +++ b/mesonbuild/compilers/mixins/clike.py @@ -27,6 +27,7 @@ import itertools import os import re import subprocess +import copy import typing as T from pathlib import Path @@ -145,6 +146,8 @@ class CLikeCompiler(Compiler): self.exe_wrapper = None else: self.exe_wrapper = exe_wrapper + # Lazy initialized in get_preprocessor() + self.preprocessor: T.Optional[Compiler] = None def compiler_args(self, args: T.Optional[T.Iterable[str]] = None) -> CLikeCompilerArgs: # This is correct, mypy just doesn't understand co-operative inheritance @@ -1328,3 +1331,18 @@ class CLikeCompiler(Compiler): def get_disable_assert_args(self) -> T.List[str]: return ['-DNDEBUG'] + + @functools.lru_cache(maxsize=None) + def can_compile(self, src: 'mesonlib.FileOrString') -> bool: + # Files we preprocess can be anything, e.g. .in + if self.mode == 'PREPROCESSOR': + return True + return super().can_compile(src) + + def get_preprocessor(self) -> Compiler: + if not self.preprocessor: + self.preprocessor = copy.copy(self) + self.preprocessor.exelist = self.exelist + self.get_preprocess_to_file_args() + self.preprocessor.mode = 'PREPROCESSOR' + self.modes.append(self.preprocessor) + return self.preprocessor diff --git a/mesonbuild/compilers/mixins/gnu.py b/mesonbuild/compilers/mixins/gnu.py index 11efcc9fe..eb1c534e6 100644 --- a/mesonbuild/compilers/mixins/gnu.py +++ b/mesonbuild/compilers/mixins/gnu.py @@ -318,6 +318,12 @@ class GnuLikeCompiler(Compiler, metaclass=abc.ABCMeta): def get_coverage_args(self) -> T.List[str]: return ['--coverage'] + def get_preprocess_to_file_args(self) -> T.List[str]: + # We want to allow preprocessing files with any extension, such as + # foo.c.in. In that case we need to tell GCC/CLANG to treat them as + # assembly file. + return self.get_preprocess_only_args() + ['-x', 'assembler-with-cpp'] + class GnuCompiler(GnuLikeCompiler): """ diff --git a/mesonbuild/compilers/mixins/visualstudio.py b/mesonbuild/compilers/mixins/visualstudio.py index dc5e96241..76ce9c18b 100644 --- a/mesonbuild/compilers/mixins/visualstudio.py +++ b/mesonbuild/compilers/mixins/visualstudio.py @@ -159,6 +159,9 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta): def get_preprocess_only_args(self) -> T.List[str]: return ['/EP'] + def get_preprocess_to_file_args(self) -> T.List[str]: + return ['/EP', '/P'] + def get_compile_only_args(self) -> T.List[str]: return ['/c'] @@ -173,6 +176,8 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta): return ['/fsanitize=address'] def get_output_args(self, target: str) -> T.List[str]: + if self.mode == 'PREPROCESSOR': + return ['/Fi' + target] if target.endswith('.exe'): return ['/Fe' + target] return ['/Fo' + target] diff --git a/mesonbuild/interpreter/compiler.py b/mesonbuild/interpreter/compiler.py index b46e502f3..7397321ed 100644 --- a/mesonbuild/interpreter/compiler.py +++ b/mesonbuild/interpreter/compiler.py @@ -4,6 +4,7 @@ import enum import functools +import os import typing as T from .. import build @@ -79,6 +80,11 @@ if T.TYPE_CHECKING: header_prefix: str header_required: T.Union[bool, coredata.UserFeatureOption] + class PreprocessKW(TypedDict): + output: str + compile_args: T.List[str] + include_directories: T.List[build.IncludeDirs] + class _TestMode(enum.Enum): @@ -184,6 +190,7 @@ class CompilerHolder(ObjectHolder['Compiler']): 'first_supported_link_argument': self.first_supported_link_argument_method, 'symbols_have_underscore_prefix': self.symbols_have_underscore_prefix_method, 'get_argument_syntax': self.get_argument_syntax_method, + 'preprocess': self.preprocess_method, }) @property @@ -734,3 +741,34 @@ class CompilerHolder(ObjectHolder['Compiler']): @noKwargs def get_argument_syntax_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> str: return self.compiler.get_argument_syntax() + + @FeatureNew('compiler.preprocess', '0.64.0') + @typed_pos_args('compiler.preprocess', varargs=(mesonlib.File, str), min_varargs=1) + @typed_kwargs( + 'compiler.preprocess', + KwargInfo('output', str, default='@PLAINNAME@.i'), + KwargInfo('compile_args', ContainerTypeInfo(list, str), listify=True, default=[]), + _INCLUDE_DIRS_KW, + ) + def preprocess_method(self, args: T.Tuple[T.List['mesonlib.FileOrString']], kwargs: 'PreprocessKW') -> T.List[build.CustomTargetIndex]: + compiler = self.compiler.get_preprocessor() + sources = self.interpreter.source_strings_to_files(args[0]) + tg_kwargs = { + f'{self.compiler.language}_args': kwargs['compile_args'], + 'build_by_default': False, + 'include_directories': kwargs['include_directories'], + } + tg = build.CompileTarget( + 'preprocessor', + self.interpreter.subdir, + self.subproject, + self.environment, + sources, + kwargs['output'], + compiler, + tg_kwargs) + self.interpreter.add_target(tg.name, tg) + # Expose this target as list of its outputs, so user can pass them to + # other targets, list outputs, etc. + private_dir = os.path.relpath(self.interpreter.backend.get_target_private_dir(tg), self.interpreter.subdir) + return [build.CustomTargetIndex(tg, os.path.join(private_dir, o)) for o in tg.outputs] diff --git a/test cases/common/255 preprocess/bar.c b/test cases/common/255 preprocess/bar.c new file mode 100644 index 000000000..43737b940 --- /dev/null +++ b/test cases/common/255 preprocess/bar.c @@ -0,0 +1,3 @@ +int bar(void) { + return BAR; +} diff --git a/test cases/common/255 preprocess/foo.c b/test cases/common/255 preprocess/foo.c new file mode 100644 index 000000000..c9d16c5eb --- /dev/null +++ b/test cases/common/255 preprocess/foo.c @@ -0,0 +1 @@ +#include diff --git a/test cases/common/255 preprocess/foo.h b/test cases/common/255 preprocess/foo.h new file mode 100644 index 000000000..ba60bf396 --- /dev/null +++ b/test cases/common/255 preprocess/foo.h @@ -0,0 +1,2 @@ +int bar(void); +int main(void) { return FOO + bar(); } diff --git a/test cases/common/255 preprocess/meson.build b/test cases/common/255 preprocess/meson.build new file mode 100644 index 000000000..4824598e7 --- /dev/null +++ b/test cases/common/255 preprocess/meson.build @@ -0,0 +1,15 @@ +project('preprocess', 'c') + +cc = meson.get_compiler('c') + +add_project_arguments(['-DFOO=0', '-DBAR=0'], language: 'c') + +pp_files = cc.preprocess('foo.c', 'bar.c', output: '@PLAINNAME@') + +foreach f : pp_files + message(f.full_path()) +endforeach + +subdir('src') + +test('test-foo', executable('app', pp_files, link_depends: file_map)) diff --git a/test cases/common/255 preprocess/src/file.map.in b/test cases/common/255 preprocess/src/file.map.in new file mode 100644 index 000000000..152fb6561 --- /dev/null +++ b/test cases/common/255 preprocess/src/file.map.in @@ -0,0 +1,3 @@ +#if 1 +Hello World +#endif diff --git a/test cases/common/255 preprocess/src/meson.build b/test cases/common/255 preprocess/src/meson.build new file mode 100644 index 000000000..4cd955444 --- /dev/null +++ b/test cases/common/255 preprocess/src/meson.build @@ -0,0 +1,3 @@ +file_map = cc.preprocess('file.map.in', + output: '@BASENAME@', +)