Merge pull request #8013 from mesonbuild/cppmodules

C++ module support

mesonbuild/backend/ninjabackend.py

@@ -112,7 +112,7 @@ rsp_threshold = get_rsp_threshold()
 # variables (or variables we use them in) is interpreted directly by ninja
 # (e.g. the value of the depfile variable is a pathname that ninja will read
 # from, etc.), so it must not be shell quoted.
-raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep'}
+raw_names = {'DEPFILE_UNQUOTED', 'DESC', 'pool', 'description', 'targetdep', 'dyndep'}
 
 NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]")
 NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]")
@@ -134,6 +134,11 @@ Please report this error with a test case to the Meson bug tracker.'''.format(te
         raise MesonException(errmsg)
     return quote_re.sub(r'$\g<0>', text)
 
+class TargetDependencyScannerInfo:
+    def __init__(self, private_dir: str, source2object: T.Dict[str, str]):
+        self.private_dir = private_dir
+        self.source2object = source2object
+
 @unique
 class Quoting(Enum):
     both = 0
@@ -683,10 +688,17 @@ int dummy;
         return False
 
     def generate_target(self, target):
+        try:
+            if isinstance(target, build.BuildTarget):
+                os.makedirs(self.get_target_private_dir_abs(target))
+        except FileExistsError:
+            pass
         if isinstance(target, build.CustomTarget):
             self.generate_custom_target(target)
         if isinstance(target, build.RunTarget):
             self.generate_run_target(target)
+        compiled_sources = []
+        source2object = {}
         name = target.get_id()
         if name in self.processed_targets:
             return
@@ -784,10 +796,12 @@ int dummy;
         # because we need `header_deps` to be fully generated in the above loop.
         for src in generated_source_files:
             if self.environment.is_llvm_ir(src):
-                o = self.generate_llvm_ir_compile(target, src)
+                o, s = self.generate_llvm_ir_compile(target, src)
             else:
-                o = self.generate_single_compile(target, src, True,
+                o, s = self.generate_single_compile(target, src, True,
                                                  order_deps=header_deps)
+            compiled_sources.append(s)
+            source2object[s] = o
             obj_list.append(o)
 
         use_pch = self.environment.coredata.base_options.get('b_pch', False)
@@ -822,32 +836,84 @@ int dummy;
             # Passing 'vala' here signifies that we want the compile
             # arguments to be specialized for C code generated by
             # valac. For instance, no warnings should be emitted.
-            obj_list.append(self.generate_single_compile(target, src, 'vala', [], header_deps))
+            o, s = self.generate_single_compile(target, src, 'vala', [], header_deps)
+            obj_list.append(o)
 
         # Generate compile targets for all the pre-existing sources for this target
         for src in target_sources.values():
             if not self.environment.is_header(src):
                 if self.environment.is_llvm_ir(src):
-                    obj_list.append(self.generate_llvm_ir_compile(target, src))
+                    o, s = self.generate_llvm_ir_compile(target, src)
+                    obj_list.append(o)
                 elif is_unity and self.get_target_source_can_unity(target, src):
                     abs_src = os.path.join(self.environment.get_build_dir(),
                                            src.rel_to_builddir(self.build_to_src))
                     unity_src.append(abs_src)
                 else:
-                    obj_list.append(self.generate_single_compile(target, src, False, [], header_deps))
+                    o, s = self.generate_single_compile(target, src, False, [], header_deps)
+                    obj_list.append(o)
+                    compiled_sources.append(s)
+                    source2object[s] = o
+
         obj_list += self.flatten_object_list(target)
         if is_unity:
             for src in self.generate_unity_files(target, unity_src):
-                obj_list.append(self.generate_single_compile(target, src, True, unity_deps + header_deps))
+                o, s = self.generate_single_compile(target, src, True, unity_deps + header_deps)
+                obj_list.append(o)
+                compiled_sources.append(s)
+                source2object[s] = o
         linker, stdlib_args = self.determine_linker_and_stdlib_args(target)
         if isinstance(target, build.StaticLibrary) and target.prelink:
             final_obj_list = self.generate_prelink(target, obj_list)
         else:
             final_obj_list = obj_list
         elem = self.generate_link(target, outname, final_obj_list, linker, pch_objects, stdlib_args=stdlib_args)
+        self.generate_dependency_scan_target(target, compiled_sources, source2object)
         self.generate_shlib_aliases(target, self.get_target_dir(target))
         self.add_build(elem)
 
+    def should_scan_target(self, target):
+        if 'cpp' not in target.compilers:
+            return False
+        # Currently only the preview version of Visual Studio is supported.
+        cpp = target.compilers['cpp']
+        if cpp.get_id() != 'msvc':
+            return False
+        if not mesonlib.current_vs_supports_modules():
+            return False
+        if mesonlib.version_compare(cpp.version, '<19.28.28617'):
+            return False
+        if mesonlib.version_compare(self.ninja_version, '<1.10.0'):
+            return False
+        return True
+
+    def generate_dependency_scan_target(self, target, compiled_sources, source2object):
+        if not self.should_scan_target(target):
+            return
+        depscan_file = self.get_dep_scan_file_for(target)
+        pickle_base = target.name + '.dat'
+        pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/')
+        pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/')
+        rule_name = 'cppscan'
+        scan_sources = self.select_sources_to_scan(compiled_sources)
+        elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources)
+        elem.add_item('picklefile', pickle_file)
+        scaninfo = TargetDependencyScannerInfo(self.get_target_private_dir(target), source2object)
+        with open(pickle_abs, 'wb') as p:
+            pickle.dump(scaninfo, p)
+        self.add_build(elem)
+
+    def select_sources_to_scan(self, compiled_sources):
+        # in practice pick up C++ and Fortran files. If some other language
+        # requires scanning (possibly Java to deal with inner class files)
+        # then add them here.
+        selected_sources = []
+        for source in compiled_sources:
+            ext = os.path.splitext(source)[1][1:]
+            if ext in compilers.lang_suffixes['cpp']:
+                selected_sources.append(source)
+        return selected_sources
+
     def process_target_dependencies(self, target):
         for t in target.get_dependencies():
             if t.get_id() not in self.processed_targets:
@@ -1074,6 +1140,8 @@ int dummy;
         self.rules = []
         self.ruledict = {}
 
+        self.add_rule_comment(NinjaComment('Rules for module scanning.'))
+        self.generate_scanner_rules()
         self.add_rule_comment(NinjaComment('Rules for compiling.'))
         self.generate_compile_rules()
         self.add_rule_comment(NinjaComment('Rules for linking.'))
@@ -1107,6 +1175,8 @@ int dummy;
         self.build_elements.append(comment)
 
     def add_rule(self, rule):
+        if rule.name in self.ruledict:
+            raise MesonException('Tried to add rule {} twice.'.format(rule.name))
         self.rules.append(rule)
         self.ruledict[rule.name] = rule
 
@@ -1957,6 +2027,26 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         self.add_rule(NinjaRule(rule, command, [], description, deps=deps,
                                 depfile=depfile))
 
+
+    def generate_scanner_rules(self):
+        scanner_languages = {'cpp'} # Fixme, add Fortran.
+        for for_machine in MachineChoice:
+            clist = self.environment.coredata.compilers[for_machine]
+            for langname, compiler in clist.items():
+                if langname not in scanner_languages:
+                    continue
+                rulename = '{}scan'.format(langname)
+                if rulename in self.ruledict:
+                    # Scanning command is the same for native and cross compilation.
+                    continue
+                command = self.environment.get_build_command() + \
+                    ['--internal', 'depscan']
+                args = ['$picklefile', '$out', '$in']
+                description = 'Module scanner for {}.'.format(langname)
+                rule = NinjaRule(rulename, command, args, description)
+                self.add_rule(rule)
+
+
     def generate_compile_rules(self):
         for for_machine in MachineChoice:
             clist = self.environment.coredata.compilers[for_machine]
@@ -2217,7 +2307,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src)
         element.add_item('ARGS', commands)
         self.add_build(element)
-        return rel_obj
+        return (rel_obj, rel_src)
 
     def get_source_dir_include_args(self, target, compiler):
         curdir = target.get_subdir()
@@ -2439,8 +2529,22 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
             element.add_orderdep(i)
         element.add_item('DEPFILE', dep_file)
         element.add_item('ARGS', commands)
+
+        self.add_dependency_scanner_entries_to_element(target, compiler, element)
         self.add_build(element)
-        return rel_obj
+        assert(isinstance(rel_obj, str))
+        assert(isinstance(rel_src, str))
+        return (rel_obj, rel_src.replace('\\', '/'))
+
+    def add_dependency_scanner_entries_to_element(self, target, compiler, element):
+        if not self.should_scan_target(target):
+            return
+        dep_scan_file = self.get_dep_scan_file_for(target)
+        element.add_item('dyndep', dep_scan_file)
+        element.add_orderdep(dep_scan_file)
+
+    def get_dep_scan_file_for(self, target):
+        return os.path.join(self.get_target_private_dir(target), 'depscan.dd')
 
     def add_header_deps(self, target, ninja_element, header_deps):
         for d in header_deps:

mesonbuild/compilers/compilers.py

@@ -55,7 +55,7 @@ lib_suffixes = ('a', 'lib', 'dll', 'dll.a', 'dylib', 'so')  # type: T.Tuple[str,
 # This means we can't include .h headers here since they could be C, C++, ObjC, etc.
 lang_suffixes = {
     'c': ('c',),
-    'cpp': ('cpp', 'cc', 'cxx', 'c++', 'hh', 'hpp', 'ipp', 'hxx', 'ino'),
+    'cpp': ('cpp', 'cc', 'cxx', 'c++', 'hh', 'hpp', 'ipp', 'hxx', 'ino', 'ixx'),
     'cuda': ('cu',),
     # f90, f95, f03, f08 are for free-form fortran ('f90' recommended)
     # f, for, ftn, fpp are for fixed-form fortran ('f' or 'for' recommended)

mesonbuild/mesonlib.py

@@ -586,6 +586,10 @@ def detect_vcs(source_dir: T.Union[str, Path]) -> T.Optional[T.Dict[str, str]]:
                 return vcs
     return None
 
+def current_vs_supports_modules() -> bool:
+    vsver = os.environ.get('VSCMD_VER', '')
+    return vsver.startswith('16.9.0') and '-pre.' in vsver
+
 # a helper class which implements the same version ordering as RPM
 class Version:
     def __init__(self, s: str) -> None:

mesonbuild/scripts/depscan.py

@@ -0,0 +1,91 @@
+# Copyright 2020 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pathlib
+import pickle
+import re
+import typing as T
+
+from ..backend.ninjabackend import TargetDependencyScannerInfo
+
+import_re = re.compile('\w*import ([a-zA-Z0-9]+);')
+export_re = re.compile('\w*export module ([a-zA-Z0-9]+);')
+
+class DependencyScanner:
+    def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]):
+        with open(pickle_file, 'rb') as pf:
+            self.target_data = pickle.load(pf) # type: TargetDependencyScannerInfo
+        self.outfile = outfile
+        self.sources = sources
+        self.provided_by = {} # type: T.Dict[str, str]
+        self.exports = {} # type: T.Dict[str, str]
+        self.needs = {} # type: T.Dict[str, T.List[str]]
+        self.sources_with_exports = [] # type: T.List[str]
+    
+    def scan_file(self, fname: str) -> None:
+        for line in pathlib.Path(fname).read_text().split('\n'):
+            import_match = import_re.match(line)
+            export_match = export_re.match(line)
+            if import_match:
+                needed = import_match.group(1)
+                if fname in self.needs:
+                    self.needs[fname].append(needed)
+                else:
+                    self.needs[fname] = [needed]
+            if export_match:
+                exported_module = export_match.group(1)
+                if exported_module in self.provided_by:
+                    raise RuntimeError('Multiple files provide module {}.'.format(exported_module))
+                self.sources_with_exports.append(fname)
+                self.provided_by[exported_module] = fname
+                self.exports[fname] = exported_module
+
+    def objname_for(self, src: str) -> str:
+        objname = self.target_data.source2object[src]
+        assert(isinstance(objname, str))
+        return objname
+
+    def ifcname_for(self, src: str) -> str:
+        return '{}.ifc'.format(self.exports[src])
+
+    def scan(self) -> int:
+        for s in self.sources:
+            self.scan_file(s)
+        with open(self.outfile, 'w') as ofile:
+            ofile.write('ninja_dyndep_version = 1\n')
+            for src in self.sources:
+                objfilename = self.objname_for(src)
+                if src in self.sources_with_exports:
+                    ifc_entry = '| ' + self.ifcname_for(src)
+                else:
+                    ifc_entry = ''
+                if src in self.needs:
+                    # FIXME, handle all sources, not just the first one
+                    modname = self.needs[src][0]
+                    provider_src = self.provided_by[modname]
+                    provider_ifc = self.ifcname_for(provider_src)
+                    mod_dep = '| ' + provider_ifc
+                else:
+                    mod_dep = ''
+                ofile.write('build {} {}: dyndep {}\n'.format(objfilename,
+                                                              ifc_entry,
+                                                              mod_dep))
+        return 0
+
+def run(args: T.List[str]) -> int:
+    pickle_file = args[0]
+    outfile = args[1]
+    sources = args[2:]
+    scanner = DependencyScanner(pickle_file, outfile, sources)
+    return scanner.scan()

run_unittests.py

@@ -5821,6 +5821,16 @@ class WindowsTests(BasePlatformTests):
         self.init(testdir, extra_args=['-Db_vscrt=mtd'])
         sanitycheck_vscrt('/MTd')
 
+    def test_modules(self):
+        if self.backend is not Backend.ninja:
+            raise unittest.SkipTest('C++ modules only work with the Ninja backend (not {}).'.format(self.backend.name))
+        if 'VSCMD_VER' not in os.environ:
+            raise unittest.SkipTest('C++ modules is only supported with Visual Studio.')
+        if version_compare(os.environ['VSCMD_VER'], '<16.9.0'):
+            raise unittest.SkipTest('C++ modules are only supported with VS 2019 Preview or newer.')
+        self.init(os.path.join(self.unit_test_dir, '87 cpp modules'))
+        self.build()
+
 
 @unittest.skipUnless(is_osx(), "requires Darwin")
 class DarwinTests(BasePlatformTests):

test cases/unit/87 cpp modules/main.cpp

@@ -0,0 +1,7 @@
+import M0;
+#include<cstdio>
+
+int main() {
+    printf("The value is %d", func0());
+    return 0;
+}

test cases/unit/87 cpp modules/meson.build

@@ -0,0 +1,17 @@
+project('cppmodules', 'cpp', default_options: ['cpp_std=c++latest'])
+
+e = executable('modtest',
+    'main.cpp',
+    'src0.ixx',
+    'src1.ixx',
+    'src2.ixx',
+    'src3.ixx',
+    'src4.ixx',
+    'src5.ixx',
+    'src6.ixx',
+    'src7.ixx',
+    'src8.ixx',
+    'src9.ixx',
+    )
+
+test('modtest', e)

test cases/unit/87 cpp modules/src0.ixx

@@ -0,0 +1,7 @@
+export module M0;
+
+import M1;
+
+export int func0() {
+  return func1();
+}

test cases/unit/87 cpp modules/src1.ixx

@@ -0,0 +1,7 @@
+export module M1;
+
+import M2;
+
+export int func1() {
+  return func2();
+}

test cases/unit/87 cpp modules/src2.ixx

@@ -0,0 +1,7 @@
+export module M2;
+
+import M3;
+
+export int func2() {
+  return func3();
+}

test cases/unit/87 cpp modules/src3.ixx

@@ -0,0 +1,7 @@
+export module M3;
+
+import M4;
+
+export int func3() {
+  return func4();
+}

test cases/unit/87 cpp modules/src4.ixx

@@ -0,0 +1,7 @@
+export module M4;
+
+import M5;
+
+export int func4() {
+  return func5();
+}

test cases/unit/87 cpp modules/src5.ixx

@@ -0,0 +1,7 @@
+export module M5;
+
+import M6;
+
+export int func5() {
+  return func6();
+}

test cases/unit/87 cpp modules/src6.ixx

@@ -0,0 +1,7 @@
+export module M6;
+
+import M7;
+
+export int func6() {
+  return func7();
+}

test cases/unit/87 cpp modules/src7.ixx

@@ -0,0 +1,7 @@
+export module M7;
+
+import M8;
+
+export int func7() {
+  return func8();
+}

test cases/unit/87 cpp modules/src8.ixx

@@ -0,0 +1,7 @@
+export module M8;
+
+import M9;
+
+export int func8() {
+  return func9();
+}

test cases/unit/87 cpp modules/src9.ixx

@@ -0,0 +1,5 @@
+export module M9;
+
+export int func9() {
+  return 42;
+}