cargo/interpreter: Implement an interpreter for Cargo TOML

This converts a Cargo TOML file into Meson AST

Co-Authored-By: Thibault Saunier <tsaunier@igalia.com>

mesonbuild/cargo/interpreter.py

@@ -0,0 +1,451 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright © 2022-2023 Intel Corporation
+
+"""Interpreter for converting Cargo Toml definitions to Meson AST
+
+There are some notable limits here. We don't even try to convert something with
+a build.rs: there's so few limits on what Cargo allows a build.rs (basically
+none), and no good way for us to convert them. In that case, an actual meson
+port will be required.
+"""
+
+from __future__ import annotations
+import dataclasses
+import glob
+import importlib
+import itertools
+import json
+import os
+import shutil
+import typing as T
+
+from . import builder
+from . import version
+from .. import mparser
+from .._pathlib import Path
+from ..mesonlib import MesonException, Popen_safe
+
+if T.TYPE_CHECKING:
+    from types import ModuleType
+
+    from . import manifest
+    from ..environment import Environment
+
+# tomllib is present in python 3.11, before that it is a pypi module called tomli,
+# we try to import tomllib, then tomli,
+# TODO: add a fallback to toml2json?
+tomllib: T.Optional[ModuleType] = None
+toml2json: T.Optional[str] = None
+for t in ['tomllib', 'tomli']:
+    try:
+        tomllib = importlib.import_module(t)
+        break
+    except ImportError:
+        pass
+else:
+    # TODO: it would be better to use an Executable here, which could be looked
+    # up in the cross file or provided by a wrap. However, that will have to be
+    # passed in externally, since we don't have (and I don't think we should),
+    # have access to the `Environment` for that in this module.
+    toml2json = shutil.which('toml2json')
+
+
+def load_toml(filename: str) -> T.Dict[object, object]:
+    if tomllib:
+        with open(filename, 'rb') as f:
+            raw = tomllib.load(f)
+    else:
+        if toml2json is None:
+            raise MesonException('Could not find an implementation of tomllib, nor toml2json')
+
+        p, out, err = Popen_safe([toml2json, filename])
+        if p.returncode != 0:
+            raise MesonException('toml2json failed to decode output\n', err)
+
+        raw = json.loads(out)
+
+    if not isinstance(raw, dict):
+        raise MesonException("Cargo.toml isn't a dictionary? How did that happen?")
+
+    return raw
+
+
+def fixup_meson_varname(name: str) -> str:
+    """Fixup a meson variable name
+
+    :param name: The name to fix
+    :return: the fixed name
+    """
+    return name.replace('-', '_')
+
+# Pylance can figure out that these do not, in fact, overlap, but mypy can't
+@T.overload
+def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ...  # type: ignore
+
+@T.overload
+def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ...  # type: ignore
+
+@T.overload
+def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ...
+
+def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency]
+                        ) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget,
+                                     manifest.FixedDependency]:
+    """Fixup raw cargo mappings to ones more suitable for python to consume.
+
+    This does the following:
+    * replaces any `-` with `_`, cargo likes the former, but python dicts make
+      keys with `-` in them awkward to work with
+    * Convert Dependndency versions from the cargo format to something meson
+      understands
+
+    :param d: The mapping to fix
+    :return: the fixed string
+    """
+    raw = {fixup_meson_varname(k): v for k, v in d.items()}
+    if 'version' in raw:
+        assert isinstance(raw['version'], str), 'for mypy'
+        raw['version'] = version.convert(raw['version'])
+    return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw)
+
+
+@dataclasses.dataclass
+class Package:
+
+    """Representation of a Cargo Package entry, with defaults filled in."""
+
+    name: str
+    version: str
+    description: str
+    resolver: T.Optional[str] = None
+    authors: T.List[str] = dataclasses.field(default_factory=list)
+    edition: manifest.EDITION = '2015'
+    rust_version: T.Optional[str] = None
+    documentation: T.Optional[str] = None
+    readme: T.Optional[str] = None
+    homepage: T.Optional[str] = None
+    repository: T.Optional[str] = None
+    license: T.Optional[str] = None
+    license_file: T.Optional[str] = None
+    keywords: T.List[str] = dataclasses.field(default_factory=list)
+    categories: T.List[str] = dataclasses.field(default_factory=list)
+    workspace: T.Optional[str] = None
+    build: T.Optional[str] = None
+    links: T.Optional[str] = None
+    exclude: T.List[str] = dataclasses.field(default_factory=list)
+    include: T.List[str] = dataclasses.field(default_factory=list)
+    publish: bool = True
+    metadata: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict)
+    default_run: T.Optional[str] = None
+    autobins: bool = True
+    autoexamples: bool = True
+    autotests: bool = True
+    autobenches: bool = True
+
+
+@dataclasses.dataclass
+class Dependency:
+
+    """Representation of a Cargo Dependency Entry."""
+
+    version: T.List[str]
+    registry: T.Optional[str] = None
+    git: T.Optional[str] = None
+    branch: T.Optional[str] = None
+    rev: T.Optional[str] = None
+    path: T.Optional[str] = None
+    optional: bool = False
+    package: T.Optional[str] = None
+    default_features: bool = False
+    features: T.List[str] = dataclasses.field(default_factory=list)
+
+    @classmethod
+    def from_raw(cls, raw: manifest.DependencyV) -> Dependency:
+        """Create a dependency from a raw cargo dictionary"""
+        if isinstance(raw, str):
+            return cls(version.convert(raw))
+        return cls(**_fixup_raw_mappings(raw))
+
+
+@dataclasses.dataclass
+class BuildTarget:
+
+    name: str
+    crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
+    path: dataclasses.InitVar[T.Optional[str]] = None
+
+    # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field
+    # True for lib, bin, test
+    test: bool = True
+
+    # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field
+    # True for lib
+    doctest: bool = False
+
+    # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field
+    # True for lib, bin, benchmark
+    bench: bool = True
+
+    # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field
+    # True for libraries and binaries
+    doc: bool = False
+
+    harness: bool = True
+    edition: manifest.EDITION = '2015'
+    required_features: T.List[str] = dataclasses.field(default_factory=list)
+    plugin: bool = False
+
+
+@dataclasses.dataclass
+class Library(BuildTarget):
+
+    """Representation of a Cargo Library Entry."""
+
+    doctest: bool = True
+    doc: bool = True
+    proc_macro: bool = False
+    crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
+    doc_scrape_examples: bool = True
+
+
+@dataclasses.dataclass
+class Binary(BuildTarget):
+
+    """Representation of a Cargo Bin Entry."""
+
+    doc: bool = True
+
+
+@dataclasses.dataclass
+class Test(BuildTarget):
+
+    """Representation of a Cargo Test Entry."""
+
+    bench: bool = True
+
+
+@dataclasses.dataclass
+class Benchmark(BuildTarget):
+
+    """Representation of a Cargo Benchmark Entry."""
+
+    test: bool = True
+
+
+@dataclasses.dataclass
+class Example(BuildTarget):
+
+    """Representation of a Cargo Example Entry."""
+
+    crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin'])
+
+
+@dataclasses.dataclass
+class Manifest:
+
+    """Cargo Manifest definition.
+
+    Most of these values map up to the Cargo Manifest, but with default values
+    if not provided.
+
+    Cargo subprojects can contain what Meson wants to treat as multiple,
+    interdependent, subprojects.
+
+    :param subdir: the subdirectory that this cargo project is in
+    :param path: the path within the cargo subproject.
+    """
+
+    package: Package
+    dependencies: T.Dict[str, Dependency]
+    dev_dependencies: T.Dict[str, Dependency]
+    build_dependencies: T.Dict[str, Dependency]
+    lib: Library
+    bin: T.List[Binary]
+    test: T.List[Test]
+    bench: T.List[Benchmark]
+    example: T.List[Example]
+    features: T.Dict[str, T.List[str]]
+    target: T.Dict[str, T.Dict[str, Dependency]]
+    subdir: str
+    path: str = ''
+
+
+def _create_project(package: Package, build: builder.Builder, env: Environment) -> mparser.FunctionNode:
+    """Create a function call
+
+    :param package: The Cargo package to generate from
+    :param filename: The full path to the file
+    :param meson_version: The generating meson version
+    :return: a FunctionNode
+    """
+    args: T.List[mparser.BaseNode] = []
+    args.extend([
+        build.string(package.name),
+        build.string('rust'),
+    ])
+    kwargs: T.Dict[str, mparser.BaseNode] = {
+        'version': build.string(package.version),
+        # Always assume that the generated meson is using the latest features
+        # This will warn when when we generate deprecated code, which is helpful
+        # for the upkeep of the module
+        'meson_version': build.string(f'>= {env.coredata.version}'),
+        'default_options': build.array([build.string(f'rust_std={package.edition}')]),
+    }
+    if package.license:
+        kwargs['license'] = build.string(package.license)
+    elif package.license_file:
+        kwargs['license_files'] = build.string(package.license_file)
+
+    return build.function('project', args, kwargs)
+
+
+def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest:
+    # This cast is a bit of a hack to deal with proc-macro
+    lib = _fixup_raw_mappings(raw_manifest.get('lib', {}))
+
+    # We need to set the name field if it's not set manually,
+    # including if other fields are set in the lib section
+    lib.setdefault('name', raw_manifest['package']['name'])
+
+    pkg = T.cast('manifest.FixedPackage',
+                 {fixup_meson_varname(k): v for k, v in raw_manifest['package'].items()})
+
+    return Manifest(
+        Package(**pkg),
+        {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dependencies', {}).items()},
+        {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dev-dependencies', {}).items()},
+        {k: Dependency.from_raw(v) for k, v in raw_manifest.get('build-dependencies', {}).items()},
+        Library(**lib),
+        [Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})],
+        [Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})],
+        [Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})],
+        [Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})],
+        raw_manifest.get('features', {}),
+        {k: {k2: Dependency.from_raw(v2) for k2, v2 in v['dependencies'].items()}
+         for k, v in raw_manifest.get('target', {}).items()},
+        subdir,
+        path,
+    )
+
+
+def _load_manifests(subdir: str) -> T.Dict[str, Manifest]:
+    filename = os.path.join(subdir, 'Cargo.toml')
+    raw = load_toml(filename)
+
+    manifests: T.Dict[str, Manifest] = {}
+
+    raw_manifest: T.Union[manifest.Manifest, manifest.VirtualManifest]
+    if 'package' in raw:
+        raw_manifest = T.cast('manifest.Manifest', raw)
+        manifest_ = _convert_manifest(raw_manifest, subdir)
+        manifests[manifest_.package.name] = manifest_
+    else:
+        raw_manifest = T.cast('manifest.VirtualManifest', raw)
+
+    if 'workspace' in raw_manifest:
+        # XXX: need to verify that python glob and cargo globbing are the
+        # same and probably write  a glob implementation. Blarg
+
+        # We need to chdir here to make the glob work correctly
+        pwd = os.getcwd()
+        os.chdir(subdir)
+        members: T.Iterable[str]
+        try:
+            members = itertools.chain.from_iterable(
+                glob.glob(m) for m in raw_manifest['workspace']['members'])
+        finally:
+            os.chdir(pwd)
+        if 'exclude' in raw_manifest['workspace']:
+            members = (x for x in members if x not in raw_manifest['workspace']['exclude'])
+
+        for m in members:
+            filename = os.path.join(subdir, m, 'Cargo.toml')
+            raw = load_toml(filename)
+
+            raw_manifest = T.cast('manifest.Manifest', raw)
+            man = _convert_manifest(raw_manifest, subdir, m)
+            manifests[man.package.name] = man
+
+    return manifests
+
+
+def load_all_manifests(subproject_dir: str) -> T.Dict[str, Manifest]:
+    """Find all cargo subprojects, and load them
+
+    :param subproject_dir: Directory to look for subprojects in
+    :return: A dictionary of rust project names to Manifests
+    """
+    manifests: T.Dict[str, Manifest] = {}
+    for p in Path(subproject_dir).iterdir():
+        if p.is_dir() and (p / 'Cargo.toml').exists():
+            manifests.update(_load_manifests(str(p)))
+    return manifests
+
+
+def _create_lib(cargo: Manifest, build: builder.Builder) -> T.List[mparser.BaseNode]:
+    kw: T.Dict[str, mparser.BaseNode] = {}
+    if cargo.dependencies:
+        ids = [build.identifier(f'dep_{n}') for n in cargo.dependencies]
+        kw['dependencies'] = build.array(
+            [build.method('get_variable', i, [build.string('dep')]) for i in ids])
+
+    # FIXME: currently assuming that an rlib is being generated, which is
+    # the most common.
+    return [
+        build.assign(
+            build.function(
+                'static_library',
+                [
+                    build.string(fixup_meson_varname(cargo.package.name)),
+                    build.string(os.path.join('src', 'lib.rs')),
+                ],
+                kw,
+            ),
+            'lib'
+        ),
+
+        build.assign(
+            build.function(
+                'declare_dependency',
+                kw={'link_with': build.identifier('lib'), **kw},
+            ),
+            'dep'
+        )
+    ]
+
+
+def interpret(cargo: Manifest, env: Environment) -> mparser.CodeBlockNode:
+    filename = os.path.join(cargo.subdir, cargo.path, 'Cargo.toml')
+    build = builder.Builder(filename)
+
+    ast: T.List[mparser.BaseNode] = [
+        _create_project(cargo.package, build, env),
+        build.assign(build.function('import', [build.string('rust')]), 'rust'),
+    ]
+
+    if cargo.dependencies:
+        for name, dep in cargo.dependencies.items():
+            kw = {
+                'version': build.array([build.string(s) for s in dep.version]),
+            }
+            ast.extend([
+                build.assign(
+                    build.method(
+                        'cargo',
+                        build.identifier('rust'),
+                        [build.string(name)],
+                        kw,
+                    ),
+                    f'dep_{fixup_meson_varname(name)}',
+                ),
+            ])
+
+    # Libs are always auto-discovered and there's no other way to handle them,
+    # which is unfortunate for reproducability
+    if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.path, 'src', 'lib.rs')):
+        ast.extend(_create_lib(cargo, build))
+
+    # XXX: make this not awful
+    block = builder.block(filename)
+    block.lines = ast
+    return block