Chiebot-Mirror
/
meson
mirror of https://github.com/mesonbuild/meson.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix DUB dependencies

Browse Source 
- fix the research of target built by DUB
 - explicitely state that DUB dynamic libraries and source libraries are not supported (yet) (mesonbuild#6581)
 - fix the build settings of recipes having sub-dependencies (mesonbuild#7560)
 - fix winlibs added from dub recipe
 - sanitization, comments, explanations...
pull/10174/head
Remi Thebault 3 years ago committed by Dylan Baker
parent 7b78c6b41b
commit 04fca24355
1 changed files with 295 additions and 126 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 421
      mesonbuild/dependencies/dub.py

421
mesonbuild/dependencies/dub.py
Unescape View File

@ -14,15 +14,15 @@
from .base import ExternalDependency, DependencyException, DependencyTypeName
from .pkgconfig import PkgConfigDependency
from ..mesonlib import Popen_safe
from ..mesonlib import (Popen_safe, OptionKey)
from ..mesonlib.universal import join_args
from ..programs import ExternalProgram
from ..compilers import DCompiler
from ..compilers.d import d_feature_args
from .. import mlog
import re
import os
import copy
import json
import platform
import typing as T
if T.TYPE_CHECKING:
@ -34,7 +34,6 @@ class DubDependency(ExternalDependency):
def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]):
super().__init__(DependencyTypeName('dub'), environment, kwargs, language='d')
self.name = name
self.module_path: T.Optional[str] = None
_temp_comp = super().get_compiler()
assert isinstance(_temp_comp, DCompiler)
@ -59,158 +58,328 @@ class DubDependency(ExternalDependency):
mlog.debug('Determining dependency {!r} with DUB executable '
'{!r}'.format(name, self.dubbin.get_path()))
# if an explicit version spec was stated, use this when querying Dub
main_pack_spec = name
if 'version' in kwargs:
version_spec = kwargs['version']
if isinstance(version_spec, list):
version_spec = " ".join(version_spec)
main_pack_spec = f'{name}@{version_spec}'
# we need to know the target architecture
arch = self.compiler.arch
dub_arch = self.compiler.arch
# we need to know the build type as well
dub_buildtype = str(environment.coredata.get_option(OptionKey('buildtype')))
# MESON types: choices=['plain', 'debug', 'debugoptimized', 'release', 'minsize', 'custom'])),
# DUB types: debug (default), plain, release, release-debug, release-nobounds, unittest, profile, profile-gc,
# docs, ddox, cov, unittest-cov, syntax and custom
if dub_buildtype == 'debugoptimized':
dub_buildtype = 'release-debug'
elif dub_buildtype == 'minsize':
dub_buildtype = 'release'
# Ask dub for the package
ret, res = self._call_dubbin(['describe', name, '--arch=' + arch])
describe_cmd = [
'describe', main_pack_spec, '--arch=' + dub_arch,
'--build=' + dub_buildtype, '--compiler=' + self.compiler.get_exelist()[-1]
]
ret, res, err = self._call_dubbin(describe_cmd)
if ret != 0:
mlog.debug('DUB describe failed: ' + err)
if 'locally' in err:
fetch_cmd = ['dub', 'fetch', main_pack_spec]
mlog.error(mlog.bold(main_pack_spec), 'is not present locally. You may try the following command:')
mlog.log(mlog.bold(join_args(fetch_cmd)))
self.is_found = False
return
comp = self.compiler.get_id().replace('llvm', 'ldc').replace('gcc', 'gdc')
packages = []
# A command that might be useful in case of missing DUB package
def dub_build_deep_command() -> str:
cmd = ['dub', 'run', 'dub-build-deep', '--yes', '--', main_pack_spec,
'--arch=' + dub_arch, '--compiler=' + self.compiler.get_exelist()[-1],
'--build=' + dub_buildtype
]
return join_args(cmd)
dub_comp_id = self.compiler.get_id().replace('llvm', 'ldc').replace('gcc', 'gdc')
description = json.loads(res)
for package in description['packages']:
packages.append(package['name'])
if package['name'] == name:
self.is_found = True
not_lib = True
if 'targetType' in package:
if package['targetType'] in ['library', 'sourceLibrary', 'staticLibrary', 'dynamicLibrary']:
not_lib = False
self.compile_args = []
self.link_args = self.raw_link_args = []
show_buildtype_warning = False
def find_package_target(pkg: T.Dict[str, str]) -> bool:
nonlocal show_buildtype_warning
# try to find a static library in a DUB folder corresponding to
# version, configuration, compiler, arch and build-type
# if can find, add to link_args.
# link_args order is meaningful, so this function MUST be called in the right order
pack_id = f'{pkg["name"]}@{pkg["version"]}'
(tgt_file, compatibilities) = self._find_compatible_package_target(description, pkg, dub_comp_id)
if tgt_file is None:
if not compatibilities:
mlog.error(mlog.bold(pack_id), 'not found')
elif 'compiler' not in compatibilities:
mlog.error(mlog.bold(pack_id), 'found but not compiled with ', mlog.bold(dub_comp_id))
elif dub_comp_id != 'gdc' and 'compiler_version' not in compatibilities:
mlog.error(mlog.bold(pack_id), 'found but not compiled with', mlog.bold(f'{dub_comp_id}-{self.compiler.version}'))
elif 'arch' not in compatibilities:
mlog.error(mlog.bold(pack_id), 'found but not compiled for', mlog.bold(dub_arch))
elif 'platform' not in compatibilities:
mlog.error(mlog.bold(pack_id), 'found but not compiled for', mlog.bold(description['platform'].join('.')))
elif 'configuration' not in compatibilities:
mlog.error(mlog.bold(pack_id), 'found but not compiled for the', mlog.bold(pkg['configuration']), 'configuration')
else:
mlog.error(mlog.bold(pack_id), 'not found')
mlog.log('You may try the following command to install the necessary DUB libraries:')
mlog.log(mlog.bold(dub_build_deep_command()))
return False
if 'build_type' not in compatibilities:
mlog.warning(mlog.bold(pack_id), 'found but not compiled as', mlog.bold(dub_buildtype))
show_buildtype_warning = True
self.link_args.append(tgt_file)
return True
# Main algorithm:
# 1. Ensure that the target is a compatible library type (not dynamic)
# 2. Find a compatible built library for the main dependency
# 3. Do the same for each sub-dependency.
# link_args MUST be in the same order than the "linkDependencies" of the main target
# 4. Add other build settings (imports, versions etc.)
# 1
self.is_found = False
packages = {}
for pkg in description['packages']:
packages[pkg['name']] = pkg
if not pkg['active']:
continue
if pkg['targetType'] == 'dynamicLibrary':
mlog.error('DUB dynamic library dependencies are not supported.')
self.is_found = False
return
## check that the main dependency is indeed a library
if pkg['name'] == name:
self.is_found = True
if not_lib:
if pkg['targetType'] not in ['library', 'sourceLibrary', 'staticLibrary']:
mlog.error(mlog.bold(name), "found but it isn't a library")
self.is_found = False
return
self.module_path = self._find_right_lib_path(package['path'], comp, description, True, package['targetFileName'])
if not os.path.exists(self.module_path):
# check if the dependency was built for other archs
archs = [['x86_64'], ['x86'], ['x86', 'x86_mscoff']]
for a in archs:
description_a = copy.deepcopy(description)
description_a['architecture'] = a
arch_module_path = self._find_right_lib_path(package['path'], comp, description_a, True, package['targetFileName'])
if arch_module_path:
mlog.error(mlog.bold(name), "found but it wasn't compiled for", mlog.bold(arch))
self.is_found = False
return
mlog.error(mlog.bold(name), "found but it wasn't compiled with", mlog.bold(comp))
self.is_found = False
return
self.version = pkg['version']
self.pkg = pkg
self.version = package['version']
self.pkg = package
# collect all targets
targets = {}
for tgt in description['targets']:
targets[tgt['rootPackage']] = tgt
if self.pkg['targetFileName'].endswith('.a'):
self.static = True
if not name in targets:
self.is_found = False
if self.pkg['targetType'] == 'sourceLibrary':
# source libraries have no associated targets,
# but some build settings like import folders must be found from the package object.
# Current algo only get these from "buildSettings" in the target object.
# Let's save this for a future PR.
# (See openssl DUB package for example of sourceLibrary)
mlog.error('DUB targets of type', mlog.bold('sourceLibrary'), 'are not supported.')
else:
mlog.error('Could not find target description for', mlog.bold(main_pack_spec))
self.compile_args = []
for flag in self.pkg['dflags']:
self.link_args.append(flag)
for path in self.pkg['importPaths']:
self.compile_args.append('-I' + os.path.join(self.pkg['path'], path))
if not self.is_found:
mlog.error(f'Could not find {name} in DUB description')
return
self.link_args = self.raw_link_args = []
for flag in self.pkg['lflags']:
self.link_args.append(flag)
# Current impl only supports static libraries
self.static = True
self.link_args.append(os.path.join(self.module_path, self.pkg['targetFileName']))
# Handle dependencies
libs = []
def add_lib_args(field_name: str, target: T.Dict[str, T.Dict[str, str]]) -> None:
if field_name in target['buildSettings']:
for lib in target['buildSettings'][field_name]:
if lib not in libs:
libs.append(lib)
if os.name != 'nt':
pkgdep = PkgConfigDependency(lib, environment, {'required': 'true', 'silent': 'true'})
for arg in pkgdep.get_compile_args():
self.compile_args.append(arg)
for arg in pkgdep.get_link_args():
self.link_args.append(arg)
for arg in pkgdep.get_link_args(raw=True):
self.raw_link_args.append(arg)
for target in description['targets']:
if target['rootPackage'] in packages:
add_lib_args('libs', target)
add_lib_args(f'libs-{platform.machine()}', target)
for file in target['buildSettings']['linkerFiles']:
lib_path = self._find_right_lib_path(file, comp, description)
if lib_path:
self.link_args.append(lib_path)
else:
self.is_found = False
def _find_right_lib_path(self,
default_path: str,
comp: str,
description: T.Dict[str, str],
folder_only: bool = False,
file_name: str = '') -> T.Optional[str]:
module_path = lib_file_name = ''
if folder_only:
module_path = default_path
lib_file_name = file_name
else:
module_path = os.path.dirname(default_path)
lib_file_name = os.path.basename(default_path)
module_build_path = os.path.join(module_path, '.dub', 'build')
# If default_path is a path to lib file and
# directory of lib don't have subdir '.dub/build'
if not os.path.isdir(module_build_path) and os.path.isfile(default_path):
if folder_only:
return module_path
else:
return default_path
# 2
if not find_package_target(self.pkg):
self.is_found = False
return
# 3
for link_dep in targets[name]['linkDependencies']:
pkg = packages[link_dep]
if not find_package_target(pkg):
self.is_found = False
return
if show_buildtype_warning:
mlog.log('If it is not suitable, try the following command and reconfigure Meson with', mlog.bold('--clearcache'))
mlog.log(mlog.bold(dub_build_deep_command()))
# 4
bs = targets[name]['buildSettings']
for flag in bs['dflags']:
self.compile_args.append(flag)
# Get D version implemented in the compiler
for path in bs['importPaths']:
self.compile_args.append('-I' + path)
for path in bs['stringImportPaths']:
if not 'import_dir' in d_feature_args[self.compiler.id]:
break
flag = d_feature_args[self.compiler.id]['import_dir']
self.compile_args.append(f'{flag}={path}')
for ver in bs['versions']:
if not 'version' in d_feature_args[self.compiler.id]:
break
flag = d_feature_args[self.compiler.id]['version']
self.compile_args.append(f'{flag}={ver}')
if bs['mainSourceFile']:
self.compile_args.append(bs['mainSourceFile'])
# pass static libraries
# linkerFiles are added during step 3
# for file in bs['linkerFiles']:
# self.link_args.append(file)
for file in bs['sourceFiles']:
# sourceFiles may contain static libraries
if file.endswith('.lib') or file.endswith('.a'):
self.link_args.append(file)
for flag in bs['lflags']:
self.link_args.append(flag)
is_windows = self.env.machines.host.is_windows()
if is_windows:
winlibs = ['kernel32', 'user32', 'gdi32', 'winspool', 'shell32', 'ole32',
'oleaut32', 'uuid', 'comdlg32', 'advapi32', 'ws2_32']
for lib in bs['libs']:
if os.name != 'nt':
# trying to add system libraries by pkg-config
pkgdep = PkgConfigDependency(lib, environment, {'required': 'true', 'silent': 'true'})
if pkgdep.is_found:
for arg in pkgdep.get_compile_args():
self.compile_args.append(arg)
for arg in pkgdep.get_link_args():
self.link_args.append(arg)
for arg in pkgdep.get_link_args(raw=True):
self.raw_link_args.append(arg)
continue
if is_windows and lib in winlibs:
self.link_args.append(lib + '.lib')
continue
# fallback
self.link_args.append('-l'+lib)
# This function finds the target of the provided JSON package, built for the right
# compiler, architecture, configuration...
# It returns (target|None, {compatibilities})
# If None is returned for target, compatibilities will list what other targets were found without full compatibility
def _find_compatible_package_target(self, jdesc: T.Dict[str, str], jpack: T.Dict[str, str], dub_comp_id: str) -> T.Tuple[str, T.Set[str]]:
dub_build_path = os.path.join(jpack['path'], '.dub', 'build')
if not os.path.exists(dub_build_path):
return (None, None)
# try to find a dir like library-debug-linux.posix-x86_64-ldc_2081-EF934983A3319F8F8FF2F0E107A363BA
# fields are:
# - configuration
# - build type
# - platform
# - architecture
# - compiler id (dmd, ldc, gdc)
# - compiler version or frontend id or frontend version?
conf = jpack['configuration']
build_type = jdesc['buildType']
platforms = jdesc['platform']
archs = jdesc['architecture']
# Get D frontend version implemented in the compiler, or the compiler version itself
# gdc doesn't support this
ret, res = self._call_dubbin(['--version'])
comp_versions = []
if ret != 0:
mlog.error('Failed to run {!r}', mlog.bold(comp))
return None
if dub_comp_id != 'gdc':
comp_versions.append(self.compiler.version)
d_ver_reg = re.search('v[0-9].[0-9][0-9][0-9].[0-9]', res) # Ex.: v2.081.2
if d_ver_reg is not None:
d_ver = d_ver_reg.group().rsplit('.', 1)[0].replace('v', '').replace('.', '') # Fix structure. Ex.: 2081
else:
d_ver = '' # gdc
ret, res = self._call_compbin(['--version'])[0:2]
if ret != 0:
mlog.error('Failed to run {!r}', mlog.bold(dub_comp_id))
return (None, None)
d_ver_reg = re.search('v[0-9].[0-9][0-9][0-9].[0-9]', res) # Ex.: v2.081.2
if d_ver_reg is not None:
frontend_version = d_ver_reg.group()
frontend_id = frontend_version.rsplit('.', 1)[0].replace('v', '').replace('.', '') # Fix structure. Ex.: 2081
comp_versions.extend([frontend_version, frontend_id])
compatibilities: T.Set[str] = set()
if not os.path.isdir(module_build_path):
return ''
# build_type is not in check_list because different build types might be compatible.
# We do show a WARNING that the build type is not the same.
# It might be critical in release builds, and acceptable otherwise
check_list = ('configuration', 'platform', 'arch', 'compiler', 'compiler_version')
for entry in os.listdir(dub_build_path):
target = os.path.join(dub_build_path, entry, jpack['targetFileName'])
if not os.path.exists(target):
# unless Dub and Meson are racing, the target file should be present
# when the directory is present
mlog.debug("WARNING: Could not find a Dub target: " + target)
continue
# we build a new set for each entry, because if this target is returned
# we want to return only the compatibilities associated to this target
# otherwise we could miss the WARNING about build_type
comps = set()
if conf in entry:
comps.add('configuration')
if build_type in entry:
comps.add('build_type')
if all(platform in entry for platform in platforms):
comps.add('platform')
if all(arch in entry for arch in archs):
comps.add('arch')
if dub_comp_id in entry:
comps.add('compiler')
if dub_comp_id == 'gdc' or any(cv in entry for cv in comp_versions):
comps.add('compiler_version')
if all(key in comps for key in check_list):
return (target, comps)
else:
compatibilities = set.union(compatibilities, comps)
# Ex.: library-debug-linux.posix-x86_64-ldc_2081-EF934983A3319F8F8FF2F0E107A363BA
build_name = '-{}-{}-{}-{}_{}'.format(description['buildType'], '.'.join(description['platform']), '.'.join(description['architecture']), comp, d_ver)
for entry in os.listdir(module_build_path):
if build_name in entry:
for file in os.listdir(os.path.join(module_build_path, entry)):
if file == lib_file_name:
if folder_only:
return os.path.join(module_build_path, entry)
else:
return os.path.join(module_build_path, entry, lib_file_name)
return (None, compatibilities)
return ''
def _call_dubbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str]:
def _call_dubbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]:
assert isinstance(self.dubbin, ExternalProgram)
p, out = Popen_safe(self.dubbin.get_command() + args, env=env)[0:2]
return p.returncode, out.strip()
p, out, err = Popen_safe(self.dubbin.get_command() + args, env=env)
return p.returncode, out.strip(), err.strip()
def _call_copmbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str]:
p, out = Popen_safe(self.compiler.get_exelist() + args, env=env)[0:2]
return p.returncode, out.strip()
def _call_compbin(self, args: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> T.Tuple[int, str, str]:
p, out, err = Popen_safe(self.compiler.get_exelist() + args, env=env)
return p.returncode, out.strip(), err.strip()
def _check_dub(self) -> T.Union[bool, ExternalProgram]:
dubbin: T.Union[bool, ExternalProgram] = ExternalProgram('dub', silent=True)

Write Preview
Loading…
Cancel
Save