Merge pull request #8162 from dcbaker/wip/2021-01/rust-module-bindgen

Add a wrapper to the rust module for bindgen

ci/ciimage/arch/install.sh

@@ -12,7 +12,7 @@ pkgs=(
   itstool gtk3 java-environment=8 gtk-doc llvm clang sdl2 graphviz
   doxygen vulkan-validation-layers openssh mercurial gtk-sharp-2 qt5-tools
   libwmf valgrind cmake netcdf-fortran openmpi nasm gnustep-base gettext
-  python-lxml hotdoc
+  python-lxml hotdoc rust-bindgen
   # cuda
 )
 

ci/ciimage/ubuntu-rolling/install.sh

@@ -27,6 +27,7 @@ pkgs=(
   libblocksruntime-dev
   libperl-dev
   liblapack-dev libscalapack-mpi-dev
+  bindgen
 )
 
 sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list"

docs/markdown/Rust-module.md

@@ -3,7 +3,7 @@ short-description: Rust language integration module
 authors:
     - name: Dylan Baker
       email: dylan@pnwbakers.com
-      years: [2020]
+      years: [2020, 2021]
 ...
 
 # Unstable Rust module
@@ -33,3 +33,51 @@ that automatically.
 
 Additional, test only dependencies may be passed via the dependencies
 argument.
+
+### bindgen(*, input: string | BuildTarget | []string | []BuildTarget, output: strng, include_directories: []include_directories, c_args: []string, args: []string)
+
+This function wraps bindgen to simplify creating rust bindings around C
+libraries. This has two advantages over hand-rolling ones own with a
+`generator` or `custom_target`:
+
+- It handles `include_directories`, so one doesn't have to manually convert them to `-I...`
+- It automatically sets up a depfile, making the results more reliable
+
+
+It takes the following keyword arguments
+
+- input — A list of Files, Strings, or CustomTargets. The first element is
+  the header bindgen will parse, additional elements are dependencies.
+- output — the name of the output rust file
+- include_directories — A list of `include_directories` objects, these are
+  passed to clang as `-I` arguments
+- c_args — A list of string arguments to pass to clang untouched
+- args — A list of string arguments to pass to `bindgen` untouched.
+
+```meson
+rust = import('unstable-rust')
+
+inc = include_directories('..'¸ '../../foo')
+
+generated = rust.bindgen(
+    'myheader.h',
+    'generated.rs',
+    include_directories : [inc, include_directories('foo'),
+    args : ['--no-rustfmt-bindings'],
+    c_args : ['-DFOO=1'],
+)
+```
+
+If the header depeneds on generated headers, those headers must be passed to
+`bindgen` as well to ensure proper dependency ordering, static headers do not
+need to be passed, as a proper depfile is generated:
+
+```meson
+h1 = custom_target(...)
+h2 = custom_target(...)
+
+r1 = rust.bindgen(
+  [h1, h2],  # h1 includes h2,
+  'out.rs',
+)
+```

docs/markdown/snippets/unstable-rust-module.md

@@ -1,4 +1,5 @@
 ## Unstable Rust module
 
 A new unstable module has been added to make using Rust with Meson easier.
-Currently it adds a single function to ease defining Rust tests.
+Currently, it adds a single function to ease defining Rust tests, as well as a
+wrapper around bindgen, making it easier to use.

mesonbuild/backend/ninjabackend.py

@@ -1597,14 +1597,19 @@ int dummy;
         args += target.get_extra_args('rust')
         args += rustc.get_output_args(os.path.join(target.subdir, target.get_filename()))
         args += self.environment.coredata.get_external_args(target.for_machine, rustc.language)
-        linkdirs = OrderedDict()
+        linkdirs = mesonlib.OrderedSet()
         for d in target.link_targets:
-            linkdirs[d.subdir] = True
+            linkdirs.add(d.subdir)
+            if d.uses_rust():
                 # specify `extern CRATE_NAME=OUTPUT_FILE` for each Rust
                 # dependency, so that collisions with libraries in rustc's
                 # sysroot don't cause ambiguity
                 args += ['--extern', '{}={}'.format(d.name, os.path.join(d.subdir, d.filename))]
-        for d in linkdirs.keys():
+            else:
+                # Rust uses -l for non rust dependencies, but we still need to add (shared|static)=foo
+                _type = 'static' if d.typename == 'static library' else 'shared'
+                args += ['-l', f'{_type}={d.name}']
+        for d in linkdirs:
             if d == '':
                 d = '.'
             args += ['-L', d]

mesonbuild/build.py

@@ -322,6 +322,13 @@ class IncludeDirs:
     def get_extra_build_dirs(self):
         return self.extra_build_dirs
 
+    def to_string_list(self, sourcedir: str) -> T.List[str]:
+        """Convert IncludeDirs object to a list of strings."""
+        strlist: T.List[str] = []
+        for idir in self.incdirs:
+            strlist.append(os.path.join(sourcedir, self.curdir, idir))
+        return strlist
+
 class ExtractedObjects:
     '''
     Holds a list of sources for which the objects must be extracted
@@ -2189,7 +2196,8 @@ class CustomTarget(Target, CommandBase):
         'env',
     ])
 
-    def __init__(self, name, subdir, subproject, kwargs, absolute_paths=False, backend=None):
+    def __init__(self, name: str, subdir: str, subproject: str, kwargs: T.Dict[str, T.Any],
+                 absolute_paths: bool = False, backend: T.Optional[str] = None):
         self.typename = 'custom'
         # TODO expose keyword arg to make MachineChoice.HOST configurable
         super().__init__(name, subdir, subproject, False, MachineChoice.HOST)
@@ -2204,7 +2212,7 @@ class CustomTarget(Target, CommandBase):
         for k in kwargs:
             if k not in CustomTarget.known_kwargs:
                 unknowns.append(k)
-        if len(unknowns) > 0:
+        if unknowns:
             mlog.warning('Unknown keyword arguments in target {}: {}'.format(self.name, ', '.join(unknowns)))
 
     def get_default_install_dir(self, environment):

mesonbuild/interpreter.py

@@ -918,7 +918,7 @@ class CustomTargetIndexHolder(TargetHolder):
         return self.interpreter.backend.get_target_filename_abs(self.held_object)
 
 class CustomTargetHolder(TargetHolder):
-    def __init__(self, target, interp):
+    def __init__(self, target: 'build.CustomTarget', interp: 'Interpreter'):
         super().__init__(target, interp)
         self.methods.update({'full_path': self.full_path_method,
                              'to_list': self.to_list_method,
@@ -1132,9 +1132,7 @@ class CompilerHolder(InterpreterObject):
         for i in incdirs:
             if not isinstance(i, IncludeDirsHolder):
                 raise InterpreterException('Include directories argument must be an include_directories object.')
-            for idir in i.held_object.get_incdirs():
-                idir = os.path.join(self.environment.get_source_dir(),
-                                    i.held_object.get_curdir(), idir)
+            for idir in i.held_object.to_string_list(self.environment.get_source_dir()):
                 args += self.compiler.get_include_args(idir, False)
         if not nobuiltins:
             opts = self.environment.coredata.options

mesonbuild/mesonlib/universal.py

@@ -1262,10 +1262,10 @@ def typeslistify(item: 'T.Union[_T, T.Sequence[_T]]',
     if isinstance(item, types):
         item = T.cast(T.List[_T], [item])
     if not isinstance(item, list):
-        raise MesonException('Item must be a list or one of {!r}'.format(types))
+        raise MesonException('Item must be a list or one of {!r}, not {!r}'.format(types, type(item)))
     for i in item:
         if i is not None and not isinstance(i, types):
-            raise MesonException('List item must be one of {!r}'.format(types))
+            raise MesonException('List item must be one of {!r}, not {!r}'.format(types, type(i)))
     return item
 
 

mesonbuild/modules/unstable_rust.py

@@ -12,18 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import typing as T
 
 from . import ExtensionModule, ModuleReturnValue
 from .. import mlog
-from ..build import BuildTarget, Executable, InvalidArguments
+from ..build import BuildTarget, CustomTargetIndex, Executable, GeneratedList, InvalidArguments, IncludeDirs, CustomTarget
 from ..dependencies import Dependency, ExternalLibrary
-from ..interpreter import ExecutableHolder, BuildTargetHolder, permitted_kwargs
+from ..interpreter import ExecutableHolder, BuildTargetHolder, CustomTargetHolder, permitted_kwargs, noPosargs
 from ..interpreterbase import InterpreterException, permittedKwargs, FeatureNew, typed_pos_args
-from ..mesonlib import stringlistify, unholder, listify
+from ..mesonlib import stringlistify, unholder, listify, typeslistify, File
 
 if T.TYPE_CHECKING:
     from ..interpreter import ModuleState, Interpreter
+    from ..dependencies import ExternalProgram
 
 
 class RustModule(ExtensionModule):
@@ -33,6 +35,7 @@ class RustModule(ExtensionModule):
     @FeatureNew('rust module', '0.57.0')
     def __init__(self, interpreter: 'Interpreter') -> None:
         super().__init__(interpreter)
+        self._bindgen_bin: T.Optional['ExternalProgram'] = None
 
     @permittedKwargs(permitted_kwargs['test'] | {'dependencies'} ^ {'protocol'})
     @typed_pos_args('rust.test', str, BuildTargetHolder)
@@ -127,6 +130,77 @@ class RustModule(ExtensionModule):
 
         return ModuleReturnValue([], [e, test])
 
+    @noPosargs
+    @permittedKwargs({'input', 'output', 'include_directories', 'c_args', 'args'})
+    def bindgen(self, state: 'ModuleState', args: T.List, kwargs: T.Dict[str, T.Any]) -> ModuleReturnValue:
+        """Wrapper around bindgen to simplify it's use.
+
+        The main thing this simplifies is the use of `include_directory`
+        objects, instead of having to pass a plethora of `-I` arguments.
+        """
+        header: T.Union[File, CustomTarget, GeneratedList, CustomTargetIndex]
+        _deps: T.Sequence[T.Union[File, CustomTarget, GeneratedList, CustomTargetIndex]]
+        try:
+            header, *_deps = unholder(self.interpreter.source_strings_to_files(listify(kwargs['input'])))
+        except KeyError:
+            raise InvalidArguments('rustmod.bindgen() `input` argument must have at least one element.')
+
+        try:
+            output: str = kwargs['output']
+        except KeyError:
+            raise InvalidArguments('rustmod.bindgen() `output` must be provided')
+        if not isinstance(output, str):
+            raise InvalidArguments('rustmod.bindgen() `output` argument must be a string.')
+
+        include_dirs: T.List[IncludeDirs] = typeslistify(unholder(listify(kwargs.get('include_directories', []))), IncludeDirs)
+        c_args: T.List[str] = stringlistify(listify(kwargs.get('c_args', [])))
+        bind_args: T.List[str] = stringlistify(listify(kwargs.get('args', [])))
+
+        # Split File and Target dependencies to add pass to CustomTarget
+        depends: T.List[BuildTarget] = []
+        depend_files: T.List[File] = []
+        for d in _deps:
+            if isinstance(d, File):
+                depend_files.append(d)
+            else:
+                depends.append(d)
+
+        inc_strs: T.List[str] = []
+        for i in include_dirs:
+            # bindgen always uses clang, so it's safe to hardcode -I here
+            inc_strs.extend([f'-I{x}' for x in i.to_string_list(state.environment.get_source_dir())])
+
+        if self._bindgen_bin is None:
+            # there's some bugs in the interpreter typeing.
+            self._bindgen_bin = T.cast('ExternalProgram', self.interpreter.find_program_impl(['bindgen']).held_object)
+
+        name: str
+        if isinstance(header, File):
+            name = header.fname
+        else:
+            name = header.get_outputs()[0]
+
+        target = CustomTarget(
+            f'rustmod-bindgen-{name}'.replace('/', '_'),
+            state.subdir,
+            state.subproject,
+            {
+                'input': header,
+                'output': output,
+                'command': self._bindgen_bin.get_command() + [
+                    '@INPUT@', '--output',
+                    os.path.join(state.environment.build_dir, '@OUTPUT@')] +
+                    bind_args + ['--'] + c_args + inc_strs +
+                    ['-MD', '-MQ', '@INPUT@', '-MF', '@DEPFILE@'],
+                'depfile': '@PLAINNAME@.d',
+                'depends': depends,
+                'depend_files': depend_files,
+            },
+            backend=state.backend,
+        )
+
+        return ModuleReturnValue([target], [CustomTargetHolder(target, self.interpreter)])
+
 
 def initialize(*args: T.List, **kwargs: T.Dict) -> RustModule:
     return RustModule(*args, **kwargs)  # type: ignore

run_project_tests.py

@@ -864,6 +864,10 @@ def skippable(suite, test):
     if test.endswith('4 qt') and mesonlib.is_osx():
         return False
 
+    # Bindgen isn't available in all distros
+    if test.endswith('12 bindgen'):
+        return False
+
     # Other framework tests are allowed to be skipped on other platforms
     return True
 

test cases/rust/12 bindgen/include/other.h

@@ -0,0 +1,6 @@
+// SPDX-license-identifer: Apache-2.0
+// Copyright © 2021 Intel Corporation
+
+# pragma once
+
+#include <stdint.h>

test cases/rust/12 bindgen/meson.build

@@ -0,0 +1,82 @@
+# SPDX-license-identifer: Apache-2.0
+# Copyright © 2021 Intel Corporation
+
+project('rustmod bindgen', ['c', 'rust'])
+
+prog_bindgen = find_program('bindgen', required : false)
+if not prog_bindgen.found()
+  error('MESON_SKIP_TEST bindgen not found')
+endif
+
+# This seems to happen on windows when libclang.dll is not in path or is not
+# valid. We must try to process a header file for this to work.
+#
+# See https://github.com/rust-lang/rust-bindgen/issues/1797
+result = run_command(prog_bindgen, 'include/other.h')
+if result.returncode() != 0
+  error('MESON_SKIP_TEST bindgen does not seem to work')
+endif
+
+# This is to test the include_directories argument to bindgen
+inc = include_directories('include')
+
+c_lib = static_library('clib', 'src/source.c', include_directories : inc)
+
+rust = import('unstable-rust')
+
+gen = rust.bindgen(
+  input : 'src/header.h',
+  output : 'header.rs',
+  include_directories : inc,
+)
+
+# see: https://github.com/mesonbuild/meson/issues/8160
+f = configure_file(
+  input : 'src/main.rs',
+  output : 'main.rs',
+  copy : true,
+)
+
+rust_bin = executable(
+  'rust_bin',
+  [f, gen],
+  link_with : c_lib,
+)
+
+test('main', rust_bin)
+
+# Test a generated header
+gen_h = custom_target(
+  'gen.h',
+  command : [find_program('src/gen_header.py'), '@INPUT@', '@OUTPUT@'],
+  output : 'gen.h',
+  input : 'src/header.h'
+)
+
+gen2_h = custom_target(
+  'other.h',
+  command : [find_program('src/gen_header.py'), '@INPUT@', '@OUTPUT@'],
+  output : 'other.h',
+  input : 'include/other.h'
+)
+
+gen_rs = rust.bindgen(
+  input : [gen_h, gen2_h],
+  output : 'gen.rs',
+)
+
+f = configure_file(
+  input : 'src/main2.rs',
+  output : 'main2.rs',
+  copy : true,
+)
+
+rust_bin2 = executable(
+  'rust_bin2',
+  [f, gen_rs],
+  link_with : c_lib,
+)
+
+test('generated header', rust_bin2)
+
+subdir('sub')

test cases/rust/12 bindgen/src/gen_header.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# SPDX-license-identifer: Apache-2.0
+# Copyright © 2021 Intel Corporation
+
+import argparse
+import shutil
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input')
+    parser.add_argument('output')
+    args = parser.parse_args()
+
+    shutil.copy2(args.input, args.output)
+
+
+if __name__ == "__main__":
+    main()

test cases/rust/12 bindgen/src/header.h

@@ -0,0 +1,8 @@
+// SPDX-license-identifer: Apache-2.0
+// Copyright © 2021 Intel Corporation
+
+#pragma once
+
+#include "other.h"
+
+int32_t add(const int32_t, const int32_t);

test cases/rust/12 bindgen/src/main.rs

@@ -0,0 +1,14 @@
+// SPDX-license-identifer: Apache-2.0
+// Copyright © 2021 Intel Corporation
+
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+
+include!("header.rs");
+
+fn main() {
+    unsafe {
+        std::process::exit(add(0, 0));
+    };
+}

test cases/rust/12 bindgen/src/main2.rs

@@ -0,0 +1,15 @@
+
+// SPDX-license-identifer: Apache-2.0
+// Copyright © 2021 Intel Corporation
+
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+
+include!("gen.rs");
+
+fn main() {
+    unsafe {
+        std::process::exit(add(0, 0));
+    };
+}

test cases/rust/12 bindgen/src/source.c

@@ -0,0 +1,8 @@
+// SPDX-license-identifer: Apache-2.0
+// Copyright © 2021 Intel Corporation
+
+#include "header.h"
+
+int32_t add(const int32_t first, const int32_t second) {
+    return first + second;
+}

test cases/rust/12 bindgen/sub/meson.build

@@ -0,0 +1,39 @@
+gen_rs3 = rust.bindgen(
+  input : [gen_h, gen2_h],
+  output : 'gen.rs',
+)
+
+f3 = configure_file(
+  input : '../src/main2.rs',
+  output : 'main3.rs',
+  copy : true,
+)
+
+rust_bin3 = executable(
+  'rust_bin3',
+  [f3, gen_rs3],
+  link_with : c_lib,
+)
+
+test('generated header (subdir)', rust_bin3)
+
+gen4 = rust.bindgen(
+  input : '../src/header.h',
+  output : 'header.rs',
+  include_directories : inc,
+)
+
+# see: https://github.com/mesonbuild/meson/issues/8160
+f4 = configure_file(
+  input : '../src/main.rs',
+  output : 'main.rs',
+  copy : true,
+)
+
+rust_bin4 = executable(
+  'rust_bin_subdir',
+  [f4, gen4],
+  link_with : c_lib,
+)
+
+test('static header (subdir)', rust_bin4)