diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 0fe8a2f85..71deab01b 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -112,7 +112,7 @@ rsp_threshold = get_rsp_threshold() # variables (or variables we use them in) is interpreted directly by ninja # (e.g. the value of the depfile variable is a pathname that ninja will read # from, etc.), so it must not be shell quoted. -raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep'} +raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep', 'dyndep'} NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]") NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]") @@ -134,6 +134,11 @@ Please report this error with a test case to the Meson bug tracker.'''.format(te raise MesonException(errmsg) return quote_re.sub(r'$\g<0>', text) +class TargetDependencyScannerInfo: + def __init__(self, private_dir: str, source2object: T.Dict[str, str]): + self.private_dir = private_dir + self.source2object = source2object + @unique class Quoting(Enum): both = 0 @@ -683,10 +688,17 @@ int dummy; return False def generate_target(self, target): + try: + if isinstance(target, build.BuildTarget): + os.makedirs(self.get_target_private_dir_abs(target)) + except FileExistsError: + pass if isinstance(target, build.CustomTarget): self.generate_custom_target(target) if isinstance(target, build.RunTarget): self.generate_run_target(target) + compiled_sources = [] + source2object = {} name = target.get_id() if name in self.processed_targets: return @@ -784,10 +796,12 @@ int dummy; # because we need `header_deps` to be fully generated in the above loop. for src in generated_source_files: if self.environment.is_llvm_ir(src): - o = self.generate_llvm_ir_compile(target, src) + o, s = self.generate_llvm_ir_compile(target, src) else: - o = self.generate_single_compile(target, src, True, + o, s = self.generate_single_compile(target, src, True, order_deps=header_deps) + compiled_sources.append(s) + source2object[s] = o obj_list.append(o) use_pch = self.environment.coredata.base_options.get('b_pch', False) @@ -822,32 +836,84 @@ int dummy; # Passing 'vala' here signifies that we want the compile # arguments to be specialized for C code generated by # valac. For instance, no warnings should be emitted. - obj_list.append(self.generate_single_compile(target, src, 'vala', [], header_deps)) + o, s = self.generate_single_compile(target, src, 'vala', [], header_deps) + obj_list.append(o) # Generate compile targets for all the pre-existing sources for this target for src in target_sources.values(): if not self.environment.is_header(src): if self.environment.is_llvm_ir(src): - obj_list.append(self.generate_llvm_ir_compile(target, src)) + o, s = self.generate_llvm_ir_compile(target, src) + obj_list.append(o) elif is_unity and self.get_target_source_can_unity(target, src): abs_src = os.path.join(self.environment.get_build_dir(), src.rel_to_builddir(self.build_to_src)) unity_src.append(abs_src) else: - obj_list.append(self.generate_single_compile(target, src, False, [], header_deps)) + o, s = self.generate_single_compile(target, src, False, [], header_deps) + obj_list.append(o) + compiled_sources.append(s) + source2object[s] = o + obj_list += self.flatten_object_list(target) if is_unity: for src in self.generate_unity_files(target, unity_src): - obj_list.append(self.generate_single_compile(target, src, True, unity_deps + header_deps)) + o, s = self.generate_single_compile(target, src, True, unity_deps + header_deps) + obj_list.append(o) + compiled_sources.append(s) + source2object[s] = o linker, stdlib_args = self.determine_linker_and_stdlib_args(target) if isinstance(target, build.StaticLibrary) and target.prelink: final_obj_list = self.generate_prelink(target, obj_list) else: final_obj_list = obj_list elem = self.generate_link(target, outname, final_obj_list, linker, pch_objects, stdlib_args=stdlib_args) + self.generate_dependency_scan_target(target, compiled_sources, source2object) self.generate_shlib_aliases(target, self.get_target_dir(target)) self.add_build(elem) + def should_scan_target(self, target): + if 'cpp' not in target.compilers: + return False + # Currently only the preview version of Visual Studio is supported. + cpp = target.compilers['cpp'] + if cpp.get_id() != 'msvc': + return False + if not mesonlib.current_vs_supports_modules(): + return False + if mesonlib.version_compare(cpp.version, '<19.28.28617'): + return False + if mesonlib.version_compare(self.ninja_version, '<1.10.0'): + return False + return True + + def generate_dependency_scan_target(self, target, compiled_sources, source2object): + if not self.should_scan_target(target): + return + depscan_file = self.get_dep_scan_file_for(target) + pickle_base = target.name + '.dat' + pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/') + pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/') + rule_name = 'cppscan' + scan_sources = self.select_sources_to_scan(compiled_sources) + elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources) + elem.add_item('picklefile', pickle_file) + scaninfo = TargetDependencyScannerInfo(self.get_target_private_dir(target), source2object) + with open(pickle_abs, 'wb') as p: + pickle.dump(scaninfo, p) + self.add_build(elem) + + def select_sources_to_scan(self, compiled_sources): + # in practice pick up C++ and Fortran files. If some other language + # requires scanning (possibly Java to deal with inner class files) + # then add them here. + selected_sources = [] + for source in compiled_sources: + ext = os.path.splitext(source)[1][1:] + if ext in compilers.lang_suffixes['cpp']: + selected_sources.append(source) + return selected_sources + def process_target_dependencies(self, target): for t in target.get_dependencies(): if t.get_id() not in self.processed_targets: @@ -1074,6 +1140,8 @@ int dummy; self.rules = [] self.ruledict = {} + self.add_rule_comment(NinjaComment('Rules for module scanning.')) + self.generate_scanner_rules() self.add_rule_comment(NinjaComment('Rules for compiling.')) self.generate_compile_rules() self.add_rule_comment(NinjaComment('Rules for linking.')) @@ -1107,6 +1175,8 @@ int dummy; self.build_elements.append(comment) def add_rule(self, rule): + if rule.name in self.ruledict: + raise MesonException('Tried to add rule {} twice.'.format(rule.name)) self.rules.append(rule) self.ruledict[rule.name] = rule @@ -1957,6 +2027,26 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) self.add_rule(NinjaRule(rule, command, [], description, deps=deps, depfile=depfile)) + + def generate_scanner_rules(self): + scanner_languages = {'cpp'} # Fixme, add Fortran. + for for_machine in MachineChoice: + clist = self.environment.coredata.compilers[for_machine] + for langname, compiler in clist.items(): + if langname not in scanner_languages: + continue + rulename = '{}scan'.format(langname) + if rulename in self.ruledict: + # Scanning command is the same for native and cross compilation. + continue + command = self.environment.get_build_command() + \ + ['--internal', 'depscan'] + args = ['$picklefile', '$out', '$in'] + description = 'Module scanner for {}.'.format(langname) + rule = NinjaRule(rulename, command, args, description) + self.add_rule(rule) + + def generate_compile_rules(self): for for_machine in MachineChoice: clist = self.environment.coredata.compilers[for_machine] @@ -2217,7 +2307,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src) element.add_item('ARGS', commands) self.add_build(element) - return rel_obj + return (rel_obj, rel_src) def get_source_dir_include_args(self, target, compiler): curdir = target.get_subdir() @@ -2439,8 +2529,22 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) element.add_orderdep(i) element.add_item('DEPFILE', dep_file) element.add_item('ARGS', commands) + + self.add_dependency_scanner_entries_to_element(target, compiler, element) self.add_build(element) - return rel_obj + assert(isinstance(rel_obj, str)) + assert(isinstance(rel_src, str)) + return (rel_obj, rel_src.replace('\\', '/')) + + def add_dependency_scanner_entries_to_element(self, target, compiler, element): + if not self.should_scan_target(target): + return + dep_scan_file = self.get_dep_scan_file_for(target) + element.add_item('dyndep', dep_scan_file) + element.add_orderdep(dep_scan_file) + + def get_dep_scan_file_for(self, target): + return os.path.join(self.get_target_private_dir(target), 'depscan.dd') def add_header_deps(self, target, ninja_element, header_deps): for d in header_deps: diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 2900e19a4..2acd429c8 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -55,7 +55,7 @@ lib_suffixes = ('a', 'lib', 'dll', 'dll.a', 'dylib', 'so') # type: T.Tuple[str, # This means we can't include .h headers here since they could be C, C++, ObjC, etc. lang_suffixes = { 'c': ('c',), - 'cpp': ('cpp', 'cc', 'cxx', 'c++', 'hh', 'hpp', 'ipp', 'hxx', 'ino'), + 'cpp': ('cpp', 'cc', 'cxx', 'c++', 'hh', 'hpp', 'ipp', 'hxx', 'ino', 'ixx'), 'cuda': ('cu',), # f90, f95, f03, f08 are for free-form fortran ('f90' recommended) # f, for, ftn, fpp are for fixed-form fortran ('f' or 'for' recommended) diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index d46dfcafa..e77314496 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -586,6 +586,10 @@ def detect_vcs(source_dir: T.Union[str, Path]) -> T.Optional[T.Dict[str, str]]: return vcs return None +def current_vs_supports_modules() -> bool: + vsver = os.environ.get('VSCMD_VER', '') + return vsver.startswith('16.9.0') and '-pre.' in vsver + # a helper class which implements the same version ordering as RPM class Version: def __init__(self, s: str) -> None: diff --git a/mesonbuild/scripts/depscan.py b/mesonbuild/scripts/depscan.py new file mode 100644 index 000000000..6eebbfe6a --- /dev/null +++ b/mesonbuild/scripts/depscan.py @@ -0,0 +1,91 @@ +# Copyright 2020 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import pickle +import re +import typing as T + +from ..backend.ninjabackend import TargetDependencyScannerInfo + +import_re = re.compile('\w*import ([a-zA-Z0-9]+);') +export_re = re.compile('\w*export module ([a-zA-Z0-9]+);') + +class DependencyScanner: + def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]): + with open(pickle_file, 'rb') as pf: + self.target_data = pickle.load(pf) # type: TargetDependencyScannerInfo + self.outfile = outfile + self.sources = sources + self.provided_by = {} # type: T.Dict[str, str] + self.exports = {} # type: T.Dict[str, str] + self.needs = {} # type: T.Dict[str, T.List[str]] + self.sources_with_exports = [] # type: T.List[str] + + def scan_file(self, fname: str) -> None: + for line in pathlib.Path(fname).read_text().split('\n'): + import_match = import_re.match(line) + export_match = export_re.match(line) + if import_match: + needed = import_match.group(1) + if fname in self.needs: + self.needs[fname].append(needed) + else: + self.needs[fname] = [needed] + if export_match: + exported_module = export_match.group(1) + if exported_module in self.provided_by: + raise RuntimeError('Multiple files provide module {}.'.format(exported_module)) + self.sources_with_exports.append(fname) + self.provided_by[exported_module] = fname + self.exports[fname] = exported_module + + def objname_for(self, src: str) -> str: + objname = self.target_data.source2object[src] + assert(isinstance(objname, str)) + return objname + + def ifcname_for(self, src: str) -> str: + return '{}.ifc'.format(self.exports[src]) + + def scan(self) -> int: + for s in self.sources: + self.scan_file(s) + with open(self.outfile, 'w') as ofile: + ofile.write('ninja_dyndep_version = 1\n') + for src in self.sources: + objfilename = self.objname_for(src) + if src in self.sources_with_exports: + ifc_entry = '| ' + self.ifcname_for(src) + else: + ifc_entry = '' + if src in self.needs: + # FIXME, handle all sources, not just the first one + modname = self.needs[src][0] + provider_src = self.provided_by[modname] + provider_ifc = self.ifcname_for(provider_src) + mod_dep = '| ' + provider_ifc + else: + mod_dep = '' + ofile.write('build {} {}: dyndep {}\n'.format(objfilename, + ifc_entry, + mod_dep)) + return 0 + +def run(args: T.List[str]) -> int: + pickle_file = args[0] + outfile = args[1] + sources = args[2:] + scanner = DependencyScanner(pickle_file, outfile, sources) + return scanner.scan() diff --git a/run_unittests.py b/run_unittests.py index b9db708a5..4b14a8732 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5821,6 +5821,16 @@ class WindowsTests(BasePlatformTests): self.init(testdir, extra_args=['-Db_vscrt=mtd']) sanitycheck_vscrt('/MTd') + def test_modules(self): + if self.backend is not Backend.ninja: + raise unittest.SkipTest('C++ modules only work with the Ninja backend (not {}).'.format(self.backend.name)) + if 'VSCMD_VER' not in os.environ: + raise unittest.SkipTest('C++ modules is only supported with Visual Studio.') + if version_compare(os.environ['VSCMD_VER'], '<16.9.0'): + raise unittest.SkipTest('C++ modules are only supported with VS 2019 Preview or newer.') + self.init(os.path.join(self.unit_test_dir, '87 cpp modules')) + self.build() + @unittest.skipUnless(is_osx(), "requires Darwin") class DarwinTests(BasePlatformTests): diff --git a/test cases/unit/87 cpp modules/main.cpp b/test cases/unit/87 cpp modules/main.cpp new file mode 100644 index 000000000..d825c7d49 --- /dev/null +++ b/test cases/unit/87 cpp modules/main.cpp @@ -0,0 +1,7 @@ +import M0; +#include + +int main() { + printf("The value is %d", func0()); + return 0; +} diff --git a/test cases/unit/87 cpp modules/meson.build b/test cases/unit/87 cpp modules/meson.build new file mode 100644 index 000000000..cdc296f43 --- /dev/null +++ b/test cases/unit/87 cpp modules/meson.build @@ -0,0 +1,17 @@ +project('cppmodules', 'cpp', default_options: ['cpp_std=c++latest']) + +e = executable('modtest', + 'main.cpp', + 'src0.ixx', + 'src1.ixx', + 'src2.ixx', + 'src3.ixx', + 'src4.ixx', + 'src5.ixx', + 'src6.ixx', + 'src7.ixx', + 'src8.ixx', + 'src9.ixx', + ) + +test('modtest', e) diff --git a/test cases/unit/87 cpp modules/src0.ixx b/test cases/unit/87 cpp modules/src0.ixx new file mode 100644 index 000000000..3ca4d1448 --- /dev/null +++ b/test cases/unit/87 cpp modules/src0.ixx @@ -0,0 +1,7 @@ +export module M0; + +import M1; + +export int func0() { + return func1(); +} diff --git a/test cases/unit/87 cpp modules/src1.ixx b/test cases/unit/87 cpp modules/src1.ixx new file mode 100644 index 000000000..cea6696de --- /dev/null +++ b/test cases/unit/87 cpp modules/src1.ixx @@ -0,0 +1,7 @@ +export module M1; + +import M2; + +export int func1() { + return func2(); +} diff --git a/test cases/unit/87 cpp modules/src2.ixx b/test cases/unit/87 cpp modules/src2.ixx new file mode 100644 index 000000000..415714cc0 --- /dev/null +++ b/test cases/unit/87 cpp modules/src2.ixx @@ -0,0 +1,7 @@ +export module M2; + +import M3; + +export int func2() { + return func3(); +} diff --git a/test cases/unit/87 cpp modules/src3.ixx b/test cases/unit/87 cpp modules/src3.ixx new file mode 100644 index 000000000..96f135c00 --- /dev/null +++ b/test cases/unit/87 cpp modules/src3.ixx @@ -0,0 +1,7 @@ +export module M3; + +import M4; + +export int func3() { + return func4(); +} diff --git a/test cases/unit/87 cpp modules/src4.ixx b/test cases/unit/87 cpp modules/src4.ixx new file mode 100644 index 000000000..1ac1a6a64 --- /dev/null +++ b/test cases/unit/87 cpp modules/src4.ixx @@ -0,0 +1,7 @@ +export module M4; + +import M5; + +export int func4() { + return func5(); +} diff --git a/test cases/unit/87 cpp modules/src5.ixx b/test cases/unit/87 cpp modules/src5.ixx new file mode 100644 index 000000000..96cf70761 --- /dev/null +++ b/test cases/unit/87 cpp modules/src5.ixx @@ -0,0 +1,7 @@ +export module M5; + +import M6; + +export int func5() { + return func6(); +} diff --git a/test cases/unit/87 cpp modules/src6.ixx b/test cases/unit/87 cpp modules/src6.ixx new file mode 100644 index 000000000..760b71cbb --- /dev/null +++ b/test cases/unit/87 cpp modules/src6.ixx @@ -0,0 +1,7 @@ +export module M6; + +import M7; + +export int func6() { + return func7(); +} diff --git a/test cases/unit/87 cpp modules/src7.ixx b/test cases/unit/87 cpp modules/src7.ixx new file mode 100644 index 000000000..8ce660801 --- /dev/null +++ b/test cases/unit/87 cpp modules/src7.ixx @@ -0,0 +1,7 @@ +export module M7; + +import M8; + +export int func7() { + return func8(); +} diff --git a/test cases/unit/87 cpp modules/src8.ixx b/test cases/unit/87 cpp modules/src8.ixx new file mode 100644 index 000000000..6a3ef96f5 --- /dev/null +++ b/test cases/unit/87 cpp modules/src8.ixx @@ -0,0 +1,7 @@ +export module M8; + +import M9; + +export int func8() { + return func9(); +} diff --git a/test cases/unit/87 cpp modules/src9.ixx b/test cases/unit/87 cpp modules/src9.ixx new file mode 100644 index 000000000..3ecb3be68 --- /dev/null +++ b/test cases/unit/87 cpp modules/src9.ixx @@ -0,0 +1,5 @@ +export module M9; + +export int func9() { + return 42; +}