# SPDX-License-Identifier: Apache-2.0 # Copyright © 2022-2023 Intel Corporation """Interpreter for converting Cargo Toml definitions to Meson AST There are some notable limits here. We don't even try to convert something with a build.rs: there's so few limits on what Cargo allows a build.rs (basically none), and no good way for us to convert them. In that case, an actual meson port will be required. """ from __future__ import annotations import dataclasses import glob import importlib import itertools import json import os import shutil import typing as T from . import builder from . import version from .._pathlib import Path from ..mesonlib import MesonException, Popen_safe if T.TYPE_CHECKING: from types import ModuleType from . import manifest from .. import mparser from ..environment import Environment # tomllib is present in python 3.11, before that it is a pypi module called tomli, # we try to import tomllib, then tomli, # TODO: add a fallback to toml2json? tomllib: T.Optional[ModuleType] = None toml2json: T.Optional[str] = None for t in ['tomllib', 'tomli']: try: tomllib = importlib.import_module(t) break except ImportError: pass else: # TODO: it would be better to use an Executable here, which could be looked # up in the cross file or provided by a wrap. However, that will have to be # passed in externally, since we don't have (and I don't think we should), # have access to the `Environment` for that in this module. toml2json = shutil.which('toml2json') def load_toml(filename: str) -> T.Dict[object, object]: if tomllib: with open(filename, 'rb') as f: raw = tomllib.load(f) else: if toml2json is None: raise MesonException('Could not find an implementation of tomllib, nor toml2json') p, out, err = Popen_safe([toml2json, filename]) if p.returncode != 0: raise MesonException('toml2json failed to decode output\n', err) raw = json.loads(out) if not isinstance(raw, dict): raise MesonException("Cargo.toml isn't a dictionary? How did that happen?") return raw def fixup_meson_varname(name: str) -> str: """Fixup a meson variable name :param name: The name to fix :return: the fixed name """ return name.replace('-', '_') # Pylance can figure out that these do not, in fact, overlap, but mypy can't @T.overload def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ... # type: ignore @T.overload def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ... # type: ignore @T.overload def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ... def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency] ) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]: """Fixup raw cargo mappings to ones more suitable for python to consume. This does the following: * replaces any `-` with `_`, cargo likes the former, but python dicts make keys with `-` in them awkward to work with * Convert Dependndency versions from the cargo format to something meson understands :param d: The mapping to fix :return: the fixed string """ raw = {fixup_meson_varname(k): v for k, v in d.items()} if 'version' in raw: assert isinstance(raw['version'], str), 'for mypy' raw['version'] = version.convert(raw['version']) return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw) @dataclasses.dataclass class Package: """Representation of a Cargo Package entry, with defaults filled in.""" name: str version: str description: str resolver: T.Optional[str] = None authors: T.List[str] = dataclasses.field(default_factory=list) edition: manifest.EDITION = '2015' rust_version: T.Optional[str] = None documentation: T.Optional[str] = None readme: T.Optional[str] = None homepage: T.Optional[str] = None repository: T.Optional[str] = None license: T.Optional[str] = None license_file: T.Optional[str] = None keywords: T.List[str] = dataclasses.field(default_factory=list) categories: T.List[str] = dataclasses.field(default_factory=list) workspace: T.Optional[str] = None build: T.Optional[str] = None links: T.Optional[str] = None exclude: T.List[str] = dataclasses.field(default_factory=list) include: T.List[str] = dataclasses.field(default_factory=list) publish: bool = True metadata: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict) default_run: T.Optional[str] = None autobins: bool = True autoexamples: bool = True autotests: bool = True autobenches: bool = True @dataclasses.dataclass class Dependency: """Representation of a Cargo Dependency Entry.""" version: T.List[str] registry: T.Optional[str] = None git: T.Optional[str] = None branch: T.Optional[str] = None rev: T.Optional[str] = None path: T.Optional[str] = None optional: bool = False package: T.Optional[str] = None default_features: bool = False features: T.List[str] = dataclasses.field(default_factory=list) @classmethod def from_raw(cls, raw: manifest.DependencyV) -> Dependency: """Create a dependency from a raw cargo dictionary""" if isinstance(raw, str): return cls(version.convert(raw)) return cls(**_fixup_raw_mappings(raw)) @dataclasses.dataclass class BuildTarget: name: str crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) path: dataclasses.InitVar[T.Optional[str]] = None # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field # True for lib, bin, test test: bool = True # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field # True for lib doctest: bool = False # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field # True for lib, bin, benchmark bench: bool = True # https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field # True for libraries and binaries doc: bool = False harness: bool = True edition: manifest.EDITION = '2015' required_features: T.List[str] = dataclasses.field(default_factory=list) plugin: bool = False @dataclasses.dataclass class Library(BuildTarget): """Representation of a Cargo Library Entry.""" doctest: bool = True doc: bool = True proc_macro: bool = False crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib']) doc_scrape_examples: bool = True @dataclasses.dataclass class Binary(BuildTarget): """Representation of a Cargo Bin Entry.""" doc: bool = True @dataclasses.dataclass class Test(BuildTarget): """Representation of a Cargo Test Entry.""" bench: bool = True @dataclasses.dataclass class Benchmark(BuildTarget): """Representation of a Cargo Benchmark Entry.""" test: bool = True @dataclasses.dataclass class Example(BuildTarget): """Representation of a Cargo Example Entry.""" crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin']) @dataclasses.dataclass class Manifest: """Cargo Manifest definition. Most of these values map up to the Cargo Manifest, but with default values if not provided. Cargo subprojects can contain what Meson wants to treat as multiple, interdependent, subprojects. :param subdir: the subdirectory that this cargo project is in :param path: the path within the cargo subproject. """ package: Package dependencies: T.Dict[str, Dependency] dev_dependencies: T.Dict[str, Dependency] build_dependencies: T.Dict[str, Dependency] lib: Library bin: T.List[Binary] test: T.List[Test] bench: T.List[Benchmark] example: T.List[Example] features: T.Dict[str, T.List[str]] target: T.Dict[str, T.Dict[str, Dependency]] subdir: str path: str = '' def _create_project(package: Package, build: builder.Builder, env: Environment) -> mparser.FunctionNode: """Create a function call :param package: The Cargo package to generate from :param filename: The full path to the file :param meson_version: The generating meson version :return: a FunctionNode """ args: T.List[mparser.BaseNode] = [] args.extend([ build.string(package.name), build.string('rust'), ]) kwargs: T.Dict[str, mparser.BaseNode] = { 'version': build.string(package.version), # Always assume that the generated meson is using the latest features # This will warn when when we generate deprecated code, which is helpful # for the upkeep of the module 'meson_version': build.string(f'>= {env.coredata.version}'), 'default_options': build.array([build.string(f'rust_std={package.edition}')]), } if package.license: kwargs['license'] = build.string(package.license) elif package.license_file: kwargs['license_files'] = build.string(package.license_file) return build.function('project', args, kwargs) def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest: # This cast is a bit of a hack to deal with proc-macro lib = _fixup_raw_mappings(raw_manifest.get('lib', {})) # We need to set the name field if it's not set manually, # including if other fields are set in the lib section lib.setdefault('name', raw_manifest['package']['name']) pkg = T.cast('manifest.FixedPackage', {fixup_meson_varname(k): v for k, v in raw_manifest['package'].items()}) return Manifest( Package(**pkg), {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dependencies', {}).items()}, {k: Dependency.from_raw(v) for k, v in raw_manifest.get('dev-dependencies', {}).items()}, {k: Dependency.from_raw(v) for k, v in raw_manifest.get('build-dependencies', {}).items()}, Library(**lib), [Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})], [Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})], [Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})], [Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})], raw_manifest.get('features', {}), {k: {k2: Dependency.from_raw(v2) for k2, v2 in v['dependencies'].items()} for k, v in raw_manifest.get('target', {}).items()}, subdir, path, ) def _load_manifests(subdir: str) -> T.Dict[str, Manifest]: filename = os.path.join(subdir, 'Cargo.toml') raw = load_toml(filename) manifests: T.Dict[str, Manifest] = {} raw_manifest: T.Union[manifest.Manifest, manifest.VirtualManifest] if 'package' in raw: raw_manifest = T.cast('manifest.Manifest', raw) manifest_ = _convert_manifest(raw_manifest, subdir) manifests[manifest_.package.name] = manifest_ else: raw_manifest = T.cast('manifest.VirtualManifest', raw) if 'workspace' in raw_manifest: # XXX: need to verify that python glob and cargo globbing are the # same and probably write a glob implementation. Blarg # We need to chdir here to make the glob work correctly pwd = os.getcwd() os.chdir(subdir) members: T.Iterable[str] try: members = itertools.chain.from_iterable( glob.glob(m) for m in raw_manifest['workspace']['members']) finally: os.chdir(pwd) if 'exclude' in raw_manifest['workspace']: members = (x for x in members if x not in raw_manifest['workspace']['exclude']) for m in members: filename = os.path.join(subdir, m, 'Cargo.toml') raw = load_toml(filename) raw_manifest = T.cast('manifest.Manifest', raw) man = _convert_manifest(raw_manifest, subdir, m) manifests[man.package.name] = man return manifests def load_all_manifests(subproject_dir: str) -> T.Dict[str, Manifest]: """Find all cargo subprojects, and load them :param subproject_dir: Directory to look for subprojects in :return: A dictionary of rust project names to Manifests """ manifests: T.Dict[str, Manifest] = {} for p in Path(subproject_dir).iterdir(): if p.is_dir() and (p / 'Cargo.toml').exists(): manifests.update(_load_manifests(str(p))) return manifests def _create_lib(cargo: Manifest, build: builder.Builder) -> T.List[mparser.BaseNode]: kw: T.Dict[str, mparser.BaseNode] = {} if cargo.dependencies: ids = [build.identifier(f'dep_{n}') for n in cargo.dependencies] kw['dependencies'] = build.array( [build.method('get_variable', i, [build.string('dep')]) for i in ids]) # FIXME: currently assuming that an rlib is being generated, which is # the most common. return [ build.assign( build.function( 'static_library', [ build.string(fixup_meson_varname(cargo.package.name)), build.string(os.path.join('src', 'lib.rs')), ], kw, ), 'lib' ), build.assign( build.function( 'declare_dependency', kw={'link_with': build.identifier('lib'), **kw}, ), 'dep' ) ] def interpret(cargo: Manifest, env: Environment) -> mparser.CodeBlockNode: filename = os.path.join(cargo.subdir, cargo.path, 'Cargo.toml') build = builder.Builder(filename) ast: T.List[mparser.BaseNode] = [ _create_project(cargo.package, build, env), build.assign(build.function('import', [build.string('rust')]), 'rust'), ] if cargo.dependencies: for name, dep in cargo.dependencies.items(): kw = { 'version': build.array([build.string(s) for s in dep.version]), } ast.extend([ build.assign( build.method( 'cargo', build.identifier('rust'), [build.string(name)], kw, ), f'dep_{fixup_meson_varname(name)}', ), ]) # Libs are always auto-discovered and there's no other way to handle them, # which is unfortunate for reproducability if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.path, 'src', 'lib.rs')): ast.extend(_create_lib(cargo, build)) # XXX: make this not awful block = builder.block(filename) block.lines = ast return block