Merge pull request #11902 from dcbaker/submit/rust-module-enhancements

Rust module enhancements for mesa

docs/markdown/Machine-files.md

@@ -237,6 +237,8 @@ section.
   subprojects. This setting has no effect if the `exe_wrapper` was not specified.
   The default value is `true`. (*new in 0.56.0*)
 - `java_home` is an absolute path pointing to the root of a Java installation.
+- `bindgen_clang_arguments` an array of extra arguments to pass to clang when
+  calling bindgen
 
 ### CMake variables
 

docs/markdown/Rust-module.md

@@ -18,7 +18,7 @@ like Meson, rather than Meson work more like rust.
 
 ## Functions
 
-### test(name: string, target: library | executable, dependencies: []Dependency)
+### test(name: string, target: library | executable, dependencies: []Dependency, link_with: []targets, rust_args: []string)
 
 This function creates a new rust unittest target from an existing rust
 based target, which may be a library or executable. It does this by
@@ -33,6 +33,11 @@ that automatically.
 Additional, test only dependencies may be passed via the dependencies
 argument.
 
+*(since 1.2.0)* the link_with argument can be used to pass additional build
+targets to link with
+*(since 1.2.0)* the `rust_args` keyword argument can be ussed to pass extra
+arguments to the Rust compiler.
+
 ### bindgen(*, input: string | BuildTarget | [](string | BuildTarget), output: string, include_directories: [](include_directories | string), c_args: []string, args: []string, dependencies: []Dependency)
 
 This function wraps bindgen to simplify creating rust bindings around C
@@ -83,10 +88,17 @@ r1 = rust.bindgen(
 )
 ```
 
-
 *Since 1.1.0* Meson will synchronize assertions for Rust and C/C++  when the
 `b_ndebug` option is set (via `-DNDEBUG` for C/C++, and `-C
 debug-assertions=on` for Rust), and will pass `-DNDEBUG` as an extra argument
 to clang. This allows for reliable wrapping of `-DNDEBUG` controlled behavior
 with `#[cfg(debug_asserions)]` and or `cfg!()`. Before 1.1.0, assertions for Rust
 were never turned on by Meson.
+
+*Since 1.2.0* Additional arguments to pass to clang may be specified in a
+*machine file in the properties section:
+
+```ini
+[properties]
+bindgen_clang_arguments = ['--target', 'x86_64-linux-gnu']
+```

docs/markdown/snippets/rust_bindegen_extra_args.md

@@ -0,0 +1,3 @@
+## rust.bindgen allows passing extra arguments to rustc
+
+This may be necessary to pass extra `cfg`s or to change warning levels.

docs/markdown/snippets/rust_extra_clang_bindgen_arguments.md

@@ -0,0 +1,8 @@
+## A machine file may be used to pass extra arguments to clang in a bindgen call
+
+Because of the way that bindgen proxies arguments to clang the only choice to
+add extra arguments currently is to wrap bindgen in a script, since the
+arguments must come after a `--`. This is inelegant, and not very portable. Now
+a `bindgen_clang_arguments` field may be placed in the machine file for the host
+machine, and these arguments will be added to every bindgen call for clang. This
+is intended to be useful for things like injecting `--target` arguments.

docs/markdown/snippets/rust_test_link_with.md

@@ -0,0 +1,4 @@
+## Add a `link_with` keyword to `rust.test()`
+
+This can already be be worked around by creating `declare_dependency()` objects
+to pass to the `dependencies` keyword, but this cuts out the middle man.

mesonbuild/backend/ninjabackend.py

@@ -1948,6 +1948,49 @@ class NinjaBackend(backends.Backend):
         args += output
         linkdirs = mesonlib.OrderedSet()
         external_deps = target.external_deps.copy()
+
+        # Have we already injected msvc-crt args?
+        #
+        # If we don't have A C, C++, or Fortran compiler that is
+        # VisualStudioLike treat this as if we've already injected them
+        #
+        # We handle this here rather than in the rust compiler because in
+        # general we don't want to link rust targets to a non-default crt.
+        # However, because of the way that MSCRTs work you can only link to one
+        # per target, so if A links to the debug one, and B links to the normal
+        # one you can't link A and B. Rust is hardcoded to the default one,
+        # so if we compile C/C++ code and link against a non-default MSCRT then
+        # linking will fail. We can work around this by injecting MSCRT link
+        # arguments early in the rustc command line
+        # https://github.com/rust-lang/rust/issues/39016
+        crt_args_injected = not any(x is not None and x.get_argument_syntax() == 'msvc' for x in
+                                    (self.environment.coredata.compilers[target.for_machine].get(l)
+                                     for l in ['c', 'cpp', 'fortran']))
+
+        crt_link_args: T.List[str] = []
+        try:
+            buildtype = self.environment.coredata.options[OptionKey('buildtype')].value
+            crt = self.environment.coredata.options[OptionKey('b_vscrt')].value
+            is_debug = buildtype == 'debug'
+
+            if crt == 'from_buildtype':
+                crt = 'mdd' if is_debug else 'md'
+            elif crt == 'static_from_buildtype':
+                crt = 'mtd' if is_debug else 'mt'
+
+            if crt == 'mdd':
+                crt_link_args = ['-l', 'static=msvcrtd']
+            elif crt == 'md':
+                # this is the default, no need to inject anything
+                crt_args_injected = True
+            elif crt == 'mtd':
+                crt_link_args = ['-l', 'static=libcmtd']
+            elif crt == 'mt':
+                crt_link_args = ['-l', 'static=libcmt']
+
+        except KeyError:
+            crt_args_injected = True
+
         # TODO: we likely need to use verbatim to handle name_prefix and name_suffix
         for d in target.link_targets:
             linkdirs.add(d.subdir)
@@ -1961,7 +2004,13 @@ class NinjaBackend(backends.Backend):
                 d_name = self._get_rust_dependency_name(target, d)
                 args += ['--extern', '{}={}'.format(d_name, os.path.join(d.subdir, d.filename))]
                 project_deps.append(RustDep(d_name, self.rust_crates[d.name].order))
-            elif isinstance(d, build.StaticLibrary):
+                continue
+
+            if not crt_args_injected and not {'c', 'cpp', 'fortran'}.isdisjoint(d.compilers):
+                args += crt_link_args
+                crt_args_injected = True
+
+            if isinstance(d, build.StaticLibrary):
                 # Rustc doesn't follow Meson's convention that static libraries
                 # are called .a, and import libraries are .lib, so we have to
                 # manually handle that.
@@ -2001,6 +2050,10 @@ class NinjaBackend(backends.Backend):
                 args += ['--extern', '{}={}'.format(d_name, os.path.join(d.subdir, d.filename))]
                 project_deps.append(RustDep(d_name, self.rust_crates[d.name].order))
             else:
+                if not crt_args_injected and not {'c', 'cpp', 'fortran'}.isdisjoint(d.compilers):
+                    crt_args_injected = True
+                    crt_args_injected = True
+
                 if rustc.linker.id in {'link', 'lld-link'}:
                     if verbatim:
                         # If we can use the verbatim modifier, then everything is great
@@ -2045,6 +2098,11 @@ class NinjaBackend(backends.Backend):
             if d == '':
                 d = '.'
             args += ['-L', d]
+
+        # Because of the way rustc links, this must come after any potential
+        # library need to link with their stdlibs (C++ and Fortran, for example)
+        args.extend(target.get_used_stdlib_args('rust'))
+
         target_deps = target.get_dependencies()
         has_shared_deps = any(isinstance(dep, build.SharedLibrary) for dep in target_deps)
         has_rust_shared_deps = any(dep.uses_rust()

mesonbuild/build.py

@@ -698,6 +698,15 @@ class BuildTarget(Target):
 
     install_dir: T.List[T.Union[str, Literal[False]]]
 
+    # This set contains all the languages a linker can link natively
+    # without extra flags. For instance, nvcc (cuda) can link C++
+    # without injecting -lc++/-lstdc++, see
+    #   https://github.com/mesonbuild/meson/issues/10570
+    _MASK_LANGS: T.FrozenSet[T.Tuple[str, str]] = frozenset([
+        # (language, linker)
+        ('cpp', 'cuda'),
+    ])
+
     def __init__(
             self,
             name: str,
@@ -1579,14 +1588,6 @@ You probably should put it in link_with instead.''')
         # Languages used by dependencies
         dep_langs = self.get_langs_used_by_deps()
 
-        # This set contains all the languages a linker can link natively
-        # without extra flags. For instance, nvcc (cuda) can link C++
-        # without injecting -lc++/-lstdc++, see
-        #   https://github.com/mesonbuild/meson/issues/10570
-        MASK_LANGS = frozenset([
-            # (language, linker)
-            ('cpp', 'cuda'),
-        ])
         # Pick a compiler based on the language priority-order
         for l in clink_langs:
             if l in self.compilers or l in dep_langs:
@@ -1597,10 +1598,7 @@ You probably should put it in link_with instead.''')
                         f'Could not get a dynamic linker for build target {self.name!r}. '
                         f'Requires a linker for language "{l}", but that is not '
                         'a project language.')
-                stdlib_args: T.List[str] = []
-                for dl in itertools.chain(self.compilers, dep_langs):
-                    if dl != linker.language and (dl, linker.language) not in MASK_LANGS:
-                        stdlib_args += all_compilers[dl].language_stdlib_only_link_flags(self.environment)
+                stdlib_args: T.List[str] = self.get_used_stdlib_args(linker.language)
                 # Type of var 'linker' is Compiler.
                 # Pretty hard to fix because the return value is passed everywhere
                 return linker, stdlib_args
@@ -1616,6 +1614,15 @@ You probably should put it in link_with instead.''')
 
         raise AssertionError(f'Could not get a dynamic linker for build target {self.name!r}')
 
+    def get_used_stdlib_args(self, link_language: str) -> T.List[str]:
+        all_compilers = self.environment.coredata.compilers[self.for_machine]
+        all_langs = set(all_compilers).union(self.get_langs_used_by_deps())
+        stdlib_args: T.List[str] = []
+        for dl in all_langs:
+            if dl != link_language and (dl, link_language) not in self._MASK_LANGS:
+                stdlib_args.extend(all_compilers[dl].language_stdlib_only_link_flags(self.environment))
+        return stdlib_args
+
     def uses_rust(self) -> bool:
         return 'rust' in self.compilers
 

mesonbuild/envconfig.py

@@ -237,6 +237,12 @@ class Properties:
         value = T.cast('T.Optional[str]', self.properties.get('java_home'))
         return Path(value) if value else None
 
+    def get_bindgen_clang_args(self) -> T.List[str]:
+        value = mesonlib.listify(self.properties.get('bindgen_clang_arguments', []))
+        if not all(isinstance(v, str) for v in value):
+            raise EnvironmentException('bindgen_clang_arguments must be a string or an array of strings')
+        return T.cast('T.List[str]', value)
+
     def __eq__(self, other: object) -> bool:
         if isinstance(other, type(self)):
             return self.properties == other.properties

mesonbuild/modules/rust.py

@@ -1,4 +1,4 @@
-# Copyright © 2020-2022 Intel Corporation
+# Copyright © 2020-2023 Intel Corporation
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,15 +18,16 @@ import typing as T
 
 from . import ExtensionModule, ModuleReturnValue, ModuleInfo
 from .. import mlog
-from ..build import BothLibraries, BuildTarget, CustomTargetIndex, Executable, ExtractedObjects, GeneratedList, IncludeDirs, CustomTarget, StructuredSources
+from ..build import BothLibraries, BuildTarget, CustomTargetIndex, Executable, ExtractedObjects, GeneratedList, IncludeDirs, CustomTarget, InvalidArguments, Jar, StructuredSources
 from ..compilers.compilers import are_asserts_disabled
 from ..dependencies import Dependency, ExternalLibrary
-from ..interpreter.type_checking import DEPENDENCIES_KW, TEST_KWS, OUTPUT_KW, INCLUDE_DIRECTORIES
+from ..interpreter.type_checking import DEPENDENCIES_KW, LINK_WITH_KW, TEST_KWS, OUTPUT_KW, INCLUDE_DIRECTORIES
 from ..interpreterbase import ContainerTypeInfo, InterpreterException, KwargInfo, typed_kwargs, typed_pos_args, noPosargs
 from ..mesonlib import File
 
 if T.TYPE_CHECKING:
     from . import ModuleState
+    from ..build import LibTypes
     from ..interpreter import Interpreter
     from ..interpreter import kwargs as _kwargs
     from ..interpreter.interpreter import SourceInputs, SourceOutputs
@@ -38,6 +39,8 @@ if T.TYPE_CHECKING:
 
         dependencies: T.List[T.Union[Dependency, ExternalLibrary]]
         is_parallel: bool
+        link_with: T.List[LibTypes]
+        rust_args: T.List[str]
 
     class FuncBindgen(TypedDict):
 
@@ -68,6 +71,14 @@ class RustModule(ExtensionModule):
         'rust.test',
         *TEST_KWS,
         DEPENDENCIES_KW,
+        LINK_WITH_KW.evolve(since='1.2.0'),
+        KwargInfo(
+            'rust_args',
+            ContainerTypeInfo(list, str),
+            listify=True,
+            default=[],
+            since='1.2.0',
+        ),
         KwargInfo('is_parallel', bool, default=False),
     )
     def test(self, state: ModuleState, args: T.Tuple[str, BuildTarget], kwargs: FuncTest) -> ModuleReturnValue:
@@ -112,6 +123,9 @@ class RustModule(ExtensionModule):
         rust.test('rust_lib_test', rust_lib)
         ```
         """
+        if any(isinstance(t, Jar) for t in kwargs.get('link_with', [])):
+            raise InvalidArguments('Rust tests cannot link with Jar targets')
+
         name = args[0]
         base_target: BuildTarget = args[1]
         if not base_target.uses_rust():
@@ -142,9 +156,11 @@ class RustModule(ExtensionModule):
         new_target_kwargs = base_target.kwargs.copy()
         # Don't mutate the shallow copied list, instead replace it with a new
         # one
-        new_target_kwargs['rust_args'] = new_target_kwargs.get('rust_args', []) + ['--test']
+        new_target_kwargs['rust_args'] = \
+            new_target_kwargs.get('rust_args', []) + kwargs['rust_args'] + ['--test']
         new_target_kwargs['install'] = False
         new_target_kwargs['dependencies'] = new_target_kwargs.get('dependencies', []) + kwargs['dependencies']
+        new_target_kwargs['link_with'] = new_target_kwargs.get('link_with', []) + kwargs['link_with']
 
         sources = T.cast('T.List[SourceOutputs]', base_target.sources.copy())
         sources.extend(base_target.generated)
@@ -194,7 +210,10 @@ class RustModule(ExtensionModule):
             else:
                 depends.append(d)
 
-        clang_args: T.List[str] = []
+        # Copy to avoid subsequent calls mutating the original
+        # TODO: if we want this to be per-machine we'll need a native kwarg
+        clang_args = state.environment.properties.host.get_bindgen_clang_args().copy()
+
         for i in state.process_include_dirs(kwargs['include_directories']):
             # bindgen always uses clang, so it's safe to hardcode -I here
             clang_args.extend([f'-I{x}' for x in i.to_string_list(

test cases/rust/20 rust and cpp/lib.cpp

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright © 2023 Intel Corporation
+
+#include "lib.hpp"
+
+#include <string>
+
+namespace {
+
+uint64_t priv_length(const std::string & str) {
+    return str.length();
+}
+
+}
+
+extern "C" uint64_t lib_length(const char * str) {
+    return priv_length(str);
+}

test cases/rust/20 rust and cpp/lib.hpp

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright © 2023 Intel Corporation
+
+#include <cstddef>
+#include <cstdint>
+
+extern "C" uint64_t lib_length(const char * str);
+

test cases/rust/20 rust and cpp/main.rs

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright © 2023 Intel Corporation
+
+use std::ffi::CString;
+use std::os::raw::c_char;
+
+extern "C" {
+        fn lib_length(s: *const c_char) -> u64;
+}
+
+fn main() {
+        let len: u64;
+        unsafe {
+                let c_str = CString::new("Hello, world!").unwrap();
+                len = lib_length(c_str.as_ptr());
+        }
+
+        std::process::exit(if len == 13 { 0 } else { 1 })
+}

test cases/rust/20 rust and cpp/meson.build

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright © 2023 Intel Corporation
+
+project(
+  'Rust and C++',
+  'rust', 'cpp',
+  default_options : ['cpp_std=c++14'],
+  meson_version : '>= 1.2.0',
+)
+
+cpplib = static_library('cpp', 'lib.cpp')
+exe = executable('main', 'main.rs', link_with : cpplib)
+
+test('main', exe)

test cases/rust/20 rust and cpp/test.json

@@ -0,0 +1,15 @@
+{
+  "matrix": {
+    "options": {
+      "b_vscrt": [
+        { "val": "none" },
+        { "val": "mdd" },
+        { "val": "md" },
+        { "val": "mtd" },
+        { "val": "mt" },
+        { "val": "from_buildtype" },
+        { "val": "static_from_buildtype" }
+      ]
+    }
+  }
+}

test cases/rust/9 unit tests/helper.rs

@@ -0,0 +1,3 @@
+pub fn subtract(a: i32, b: i32) -> i32 {
+        a - b
+}

test cases/rust/9 unit tests/meson.build

@@ -31,13 +31,17 @@ test(
   suite : ['foo'],
 )
 
-exe = executable('rust_exe', ['test2.rs', 'test.rs'])
+exe = executable('rust_exe', ['test2.rs', 'test.rs'], build_by_default : false)
 
 rust = import('unstable-rust')
 rust.test('rust_test_from_exe', exe, should_fail : true)
 
-lib = static_library('rust_static', ['test.rs'])
+lib = static_library('rust_static', ['test.rs'], build_by_default : false)
 rust.test('rust_test_from_static', lib, args: ['--skip', 'test_add_intentional_fail'])
 
-lib = shared_library('rust_shared', ['test.rs'])
+lib = shared_library('rust_shared', ['test.rs'], build_by_default : false)
 rust.test('rust_test_from_shared', lib, args: ['--skip', 'test_add_intentional_fail'])
+
+helper = static_library('helper', 'helper.rs')
+lib = static_library('rust_link_with', 'test3.rs', build_by_default : false)
+rust.test('rust_test_link_with', lib, link_with : helper, rust_args : ['--cfg', 'broken="false"'])

test cases/rust/9 unit tests/test3.rs

@@ -0,0 +1,23 @@
+pub fn add(a: i32, b: i32) -> i32 {
+    a + b
+}
+
+#[cfg(test)]
+mod tests {
+        extern crate helper;
+
+        use super::*;
+
+        // This is an intentinally broken test that should be turned off by extra rust arguments
+        #[cfg(not(broken = "false"))]
+        #[test]
+        fn test_broken() {
+                assert_eq!(0, 5);
+        }
+
+        #[test]
+        fn test_add_sub() {
+                let x = helper::subtract(6, 5);
+                assert_eq!(add(x, 5), 6);
+        }
+}

unittests/machinefiletests.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import annotations
+
 import subprocess
 import tempfile
 import textwrap
@@ -67,7 +69,7 @@ class NativeFileTests(BasePlatformTests):
         self.current_config = 0
         self.current_wrapper = 0
 
-    def helper_create_native_file(self, values):
+    def helper_create_native_file(self, values: T.Dict[str, T.Dict[str, T.Union[str, int, float, bool, T.Sequence[T.Union[str, int, float, bool]]]]]) -> str:
         """Create a config file as a temporary file.
 
         values should be a nested dictionary structure of {section: {key:
@@ -81,10 +83,10 @@ class NativeFileTests(BasePlatformTests):
                 for k, v in entries.items():
                     if isinstance(v, (bool, int, float)):
                         f.write(f"{k}={v}\n")
-                    elif isinstance(v, list):
-                        f.write("{}=[{}]\n".format(k, ', '.join([f"'{w}'" for w in v])))
-                    else:
+                    elif isinstance(v, str):
                         f.write(f"{k}='{v}'\n")
+                    else:
+                        f.write("{}=[{}]\n".format(k, ', '.join([f"'{w}'" for w in v])))
         return filename
 
     def helper_create_binary_wrapper(self, binary, dir_=None, extra_args=None, **kwargs):
@@ -622,6 +624,29 @@ class NativeFileTests(BasePlatformTests):
         else:
             self.fail('Did not find bindir in build options?')
 
+    @skip_if_not_language('rust')
+    def test_bindgen_clang_arguments(self) -> None:
+        if self.backend is not Backend.ninja:
+            raise SkipTest('Rust is only supported with Ninja')
+
+        testcase = os.path.join(self.rust_test_dir, '12 bindgen')
+        config = self.helper_create_native_file({
+            'properties': {'bindgen_clang_arguments': 'sentinal'}
+        })
+
+        self.init(testcase, extra_args=['--native-file', config])
+        targets: T.List[T.Dict[str, T.Any]] = self.introspect('--targets')
+        for t in targets:
+            if t['id'].startswith('rustmod-bindgen'):
+                args: T.List[str] = t['target_sources'][0]['compiler']
+                self.assertIn('sentinal', args, msg="Did not find machine file value")
+                cargs_start = args.index('--')
+                sent_arg = args.index('sentinal')
+                self.assertLess(cargs_start, sent_arg, msg='sentinal argument does not come after "--"')
+                break
+        else:
+            self.fail('Did not find a bindgen target')
+
 
 class CrossFileTests(BasePlatformTests):