From 7d1431a06039cb29bc2585929a9fbac58bdedb0e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 1 Oct 2021 09:38:52 -0700 Subject: [PATCH] build: plumb structured sources into BuildTargets --- mesonbuild/ast/introspection.py | 2 +- mesonbuild/build.py | 73 ++++++++++++++++++++------- mesonbuild/interpreter/interpreter.py | 37 ++++++++++++-- mesonbuild/modules/gnome.py | 16 +++--- mesonbuild/modules/unstable_rust.py | 11 ++-- unittests/allplatformstests.py | 1 + 6 files changed, 105 insertions(+), 35 deletions(-) diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 9bd73d729..af295423f 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -249,7 +249,7 @@ class IntrospectionInterpreter(AstInterpreter): objects = [] # type: T.List[T.Any] empty_sources = [] # type: T.List[T.Any] # Passing the unresolved sources list causes errors - target = targetclass(name, self.subdir, self.subproject, for_machine, empty_sources, objects, self.environment, kwargs_reduced) + target = targetclass(name, self.subdir, self.subproject, for_machine, empty_sources, [], objects, self.environment, kwargs_reduced) new_target = { 'name': target.get_basename(), diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 033941c66..f2de50c2f 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -739,14 +739,16 @@ class BuildTarget(Target): install_dir: T.List[T.Union[str, bool]] - def __init__(self, name: str, subdir: str, subproject: 'SubProject', for_machine: MachineChoice, - sources: T.List['SourceOutputs'], objects, environment: environment.Environment, kwargs): + def __init__(self, name: str, subdir: str, subproject: SubProject, for_machine: MachineChoice, + sources: T.List['SourceOutputs'], structured_sources: T.Optional[StructuredSources], + objects, environment: environment.Environment, kwargs): super().__init__(name, subdir, subproject, True, for_machine) unity_opt = environment.coredata.get_option(OptionKey('unity')) self.is_unity = unity_opt == 'on' or (unity_opt == 'subprojects' and subproject != '') self.environment = environment self.compilers = OrderedDict() # type: OrderedDict[str, Compiler] self.objects: T.List[T.Union[str, 'File', 'ExtractedObjects']] = [] + self.structured_sources = structured_sources self.external_deps: T.List[dependencies.Dependency] = [] self.include_dirs: T.List['IncludeDirs'] = [] self.link_language = kwargs.get('link_language') @@ -778,13 +780,18 @@ class BuildTarget(Target): self.process_kwargs(kwargs, environment) self.check_unknown_kwargs(kwargs) self.process_compilers() - if not any([self.sources, self.generated, self.objects, self.link_whole]): + if not any([self.sources, self.generated, self.objects, self.link_whole, self.structured_sources]): raise InvalidArguments(f'Build target {name} has no sources.') self.process_compilers_late() self.validate_sources() self.validate_install(environment) self.check_module_linking() + if self.structured_sources and any([self.sources, self.generated]): + raise MesonException('cannot mix structured sources and unstructured sources') + if self.structured_sources and 'rust' not in self.compilers: + raise MesonException('structured sources are only supported in Rust targets') + def __repr__(self): repr_str = "<{0} {1}: {2}>" return repr_str.format(self.__class__.__name__, self.get_id(), self.filename) @@ -888,21 +895,31 @@ class BuildTarget(Target): self.compilers[lang] = compilers[lang] break - def process_compilers(self): + def process_compilers(self) -> None: ''' Populate self.compilers, which is the list of compilers that this target will use for compiling all its sources. We also add compilers that were used by extracted objects to simplify dynamic linker determination. ''' - if not self.sources and not self.generated and not self.objects: + if not any([self.sources, self.generated, self.objects, self.structured_sources]): return # Populate list of compilers compilers = self.environment.coredata.compilers[self.for_machine] # Pre-existing sources - sources = list(self.sources) + sources: T.List['FileOrString'] = list(self.sources) + generated = self.generated.copy() + + if self.structured_sources: + for v in self.structured_sources.sources.values(): + for src in v: + if isinstance(src, (str, File)): + sources.append(src) + else: + generated.append(src) + # All generated sources - for gensrc in self.generated: + for gensrc in generated: for s in gensrc.get_outputs(): # Generated objects can't be compiled, so don't use them for # compiler detection. If our target only has generated objects, @@ -1626,6 +1643,16 @@ You probably should put it in link_with instead.''') elif self.generated: if self.generated[0].get_outputs()[0].endswith('.rs'): return True + elif self.structured_sources: + for v in self.structured_sources.sources.values(): + for s in v: + if isinstance(s, (str, File)): + if s.endswith('.rs'): + return True + else: + for ss in s.get_outputs(): + if ss.endswith('.rs'): + return True return False def get_using_msvc(self) -> bool: @@ -1827,12 +1854,13 @@ class Executable(BuildTarget): known_kwargs = known_exe_kwargs def __init__(self, name: str, subdir: str, subproject: str, for_machine: MachineChoice, - sources: T.List[File], objects, environment: environment.Environment, kwargs): + sources: T.List[File], structured_sources: T.Optional['StructuredSources'], + objects, environment: environment.Environment, kwargs): self.typename = 'executable' key = OptionKey('b_pie') if 'pie' not in kwargs and key in environment.coredata.options: kwargs['pie'] = environment.coredata.options[key].value - super().__init__(name, subdir, subproject, for_machine, sources, objects, environment, kwargs) + super().__init__(name, subdir, subproject, for_machine, sources, structured_sources, objects, environment, kwargs) # Unless overridden, executables have no suffix or prefix. Except on # Windows and with C#/Mono executables where the suffix is 'exe' if not hasattr(self, 'prefix'): @@ -1952,9 +1980,11 @@ class Executable(BuildTarget): class StaticLibrary(BuildTarget): known_kwargs = known_stlib_kwargs - def __init__(self, name, subdir, subproject, for_machine: MachineChoice, sources, objects, environment, kwargs): + def __init__(self, name: str, subdir: str, subproject: str, for_machine: MachineChoice, + sources: T.List[File], structured_sources: T.Optional['StructuredSources'], + objects, environment: environment.Environment, kwargs): self.typename = 'static library' - super().__init__(name, subdir, subproject, for_machine, sources, objects, environment, kwargs) + super().__init__(name, subdir, subproject, for_machine, sources, structured_sources, objects, environment, kwargs) if 'cs' in self.compilers: raise InvalidArguments('Static libraries not supported for C#.') if 'rust' in self.compilers: @@ -2013,7 +2043,9 @@ class StaticLibrary(BuildTarget): class SharedLibrary(BuildTarget): known_kwargs = known_shlib_kwargs - def __init__(self, name, subdir, subproject, for_machine: MachineChoice, sources, objects, environment, kwargs): + def __init__(self, name: str, subdir: str, subproject: str, for_machine: MachineChoice, + sources: T.List[File], structured_sources: T.Optional['StructuredSources'], + objects, environment: environment.Environment, kwargs): self.typename = 'shared library' self.soversion = None self.ltversion = None @@ -2030,7 +2062,7 @@ class SharedLibrary(BuildTarget): self.debug_filename = None # Use by the pkgconfig module self.shared_library_only = False - super().__init__(name, subdir, subproject, for_machine, sources, objects, environment, kwargs) + super().__init__(name, subdir, subproject, for_machine, sources, structured_sources, objects, environment, kwargs) if 'rust' in self.compilers: # If no crate type is specified, or it's the generic lib type, use dylib if not hasattr(self, 'rust_crate_type') or self.rust_crate_type == 'lib': @@ -2338,12 +2370,15 @@ class SharedLibrary(BuildTarget): class SharedModule(SharedLibrary): known_kwargs = known_shmod_kwargs - def __init__(self, name, subdir, subproject, for_machine: MachineChoice, sources, objects, environment, kwargs): + def __init__(self, name: str, subdir: str, subproject: str, for_machine: MachineChoice, + sources: T.List[File], structured_sources: T.Optional['StructuredSources'], + objects, environment: environment.Environment, kwargs): if 'version' in kwargs: raise MesonException('Shared modules must not specify the version kwarg.') if 'soversion' in kwargs: raise MesonException('Shared modules must not specify the soversion kwarg.') - super().__init__(name, subdir, subproject, for_machine, sources, objects, environment, kwargs) + super().__init__(name, subdir, subproject, for_machine, sources, + structured_sources, objects, environment, kwargs) self.typename = 'shared module' # We need to set the soname in cases where build files link the module # to build targets, see: https://github.com/mesonbuild/meson/issues/9492 @@ -2663,15 +2698,19 @@ class AliasTarget(RunTarget): class Jar(BuildTarget): known_kwargs = known_jar_kwargs - def __init__(self, name, subdir, subproject, for_machine: MachineChoice, sources, objects, environment, kwargs): + def __init__(self, name: str, subdir: str, subproject: str, for_machine: MachineChoice, + sources: T.List[File], structured_sources: T.Optional['StructuredSources'], + objects, environment: environment.Environment, kwargs): self.typename = 'jar' - super().__init__(name, subdir, subproject, for_machine, sources, objects, environment, kwargs) + super().__init__(name, subdir, subproject, for_machine, sources, structured_sources, objects, environment, kwargs) for s in self.sources: if not s.endswith('.java'): raise InvalidArguments(f'Jar source {s} is not a java file.') for t in self.link_targets: if not isinstance(t, Jar): raise InvalidArguments(f'Link target {t} is not a jar target.') + if self.structured_sources: + raise InvalidArguments(f'structured sources are not supported in Java targets.') self.filename = self.name + '.jar' self.outputs = [self.filename] self.java_args = kwargs.get('java_args', []) diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 6627b5e48..bc011b7c1 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -105,7 +105,7 @@ if T.TYPE_CHECKING: # Input source types passed to the build.Target classes SourceOutputs = T.Union[mesonlib.File, build.GeneratedList, build.BuildTarget, build.CustomTargetIndex, build.CustomTarget, - build.ExtractedObjects, build.GeneratedList] + build.ExtractedObjects, build.GeneratedList, build.StructuredSources] def _project_version_validator(value: T.Union[T.List, str, mesonlib.File, None]) -> T.Optional[str]: @@ -2863,7 +2863,7 @@ Try setting b_lundef to false instead.'''.format(self.coredata.options[OptionKey else: raise InterpreterException(f'Unknown default_library value: {default_library}.') - def build_target(self, node, args, kwargs, targetclass): + def build_target(self, node: mparser.BaseNode, args, kwargs, targetclass): @FeatureNewKwargs('build target', '0.42.0', ['rust_crate_type', 'build_rpath', 'implicit_include_directories']) @FeatureNewKwargs('build target', '0.41.0', ['rust_args']) @FeatureNewKwargs('build target', '0.40.0', ['build_by_default']) @@ -2896,8 +2896,39 @@ Try setting b_lundef to false instead.'''.format(self.coredata.options[OptionKey # passed to library() when default_library == 'static'. kwargs = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs} + srcs: T.List['SourceInputs'] = [] + struct: T.Optional[build.StructuredSources] = build.StructuredSources() + for s in sources: + if isinstance(s, build.StructuredSources): + struct = struct + s + else: + srcs.append(s) + + if not struct: + struct = None + else: + # Validate that we won't end up with two outputs with the same name. + # i.e, don't allow: + # [structured_sources('foo/bar.rs'), structured_sources('bar/bar.rs')] + for v in struct.sources.values(): + outputs: T.Set[str] = set() + for f in v: + o: T.List[str] + if isinstance(f, str): + o = [os.path.basename(f)] + elif isinstance(f, mesonlib.File): + o = [f.fname] + else: + o = f.get_outputs() + conflicts = outputs.intersection(o) + if conflicts: + raise InvalidArguments.from_node( + f"Conflicting sources in structured sources: {', '.join(sorted(conflicts))}", + node=node) + outputs.update(o) + kwargs['include_directories'] = self.extract_incdirs(kwargs) - target = targetclass(name, self.subdir, self.subproject, for_machine, sources, objs, self.environment, kwargs) + target = targetclass(name, self.subdir, self.subproject, for_machine, srcs, struct, objs, self.environment, kwargs) target.project_version = self.project_version self.add_stdlib_info(target) diff --git a/mesonbuild/modules/gnome.py b/mesonbuild/modules/gnome.py index 486b395a4..d1729ef7f 100644 --- a/mesonbuild/modules/gnome.py +++ b/mesonbuild/modules/gnome.py @@ -607,10 +607,10 @@ class GnomeModule(ExtensionModule): def _get_link_args(self, state: 'ModuleState', lib: T.Union[build.SharedLibrary, build.StaticLibrary], - depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']], + depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]], include_rpath: bool = False, use_gir_args: bool = False - ) -> T.Tuple[T.List[str], T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']]]: + ) -> T.Tuple[T.List[str], T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]]]: link_command: T.List[str] = [] new_depends = list(depends) # Construct link args @@ -638,12 +638,12 @@ class GnomeModule(ExtensionModule): def _get_dependencies_flags( self, deps: T.Sequence[T.Union['Dependency', build.BuildTarget, build.CustomTarget, build.CustomTargetIndex]], state: 'ModuleState', - depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']], + depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]], include_rpath: bool = False, use_gir_args: bool = False, separate_nodedup: bool = False ) -> T.Tuple[OrderedSet[str], OrderedSet[str], OrderedSet[str], T.Optional[T.List[str]], OrderedSet[str], - T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']]]: + T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]]]: cflags: OrderedSet[str] = OrderedSet() internal_ldflags: OrderedSet[str] = OrderedSet() external_ldflags: OrderedSet[str] = OrderedSet() @@ -931,7 +931,7 @@ class GnomeModule(ExtensionModule): girfile: str, scan_command: T.Sequence[T.Union['FileOrString', Executable, ExternalProgram, OverrideProgram]], generated_files: T.Sequence[T.Union[str, mesonlib.File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList]], - depends: T.Sequence[T.Union['FileOrString', build.BuildTarget, 'build.GeneratedTypes']], + depends: T.Sequence[T.Union['FileOrString', build.BuildTarget, 'build.GeneratedTypes', build.StructuredSources]], kwargs: T.Dict[str, T.Any]) -> GirTarget: install = kwargs['install_gir'] if install is None: @@ -989,8 +989,8 @@ class GnomeModule(ExtensionModule): def _gather_typelib_includes_and_update_depends( state: 'ModuleState', deps: T.Sequence[T.Union[Dependency, build.BuildTarget, build.CustomTarget, build.CustomTargetIndex]], - depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']] - ) -> T.Tuple[T.List[str], T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString']]]: + depends: T.Sequence[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]] + ) -> T.Tuple[T.List[str], T.List[T.Union[build.BuildTarget, 'build.GeneratedTypes', 'FileOrString', build.StructuredSources]]]: # Need to recursively add deps on GirTarget sources from our # dependencies and also find the include directories needed for the # typelib generation custom target below. @@ -1092,7 +1092,7 @@ class GnomeModule(ExtensionModule): srcdir = os.path.join(state.environment.get_source_dir(), state.subdir) builddir = os.path.join(state.environment.get_build_dir(), state.subdir) - depends: T.List[T.Union['FileOrString', 'build.GeneratedTypes', build.BuildTarget]] = [] + depends: T.List[T.Union['FileOrString', 'build.GeneratedTypes', build.BuildTarget, build.StructuredSources]] = [] depends.extend(gir_dep.sources) depends.extend(girtargets) diff --git a/mesonbuild/modules/unstable_rust.py b/mesonbuild/modules/unstable_rust.py index 501273f9a..f656cd85d 100644 --- a/mesonbuild/modules/unstable_rust.py +++ b/mesonbuild/modules/unstable_rust.py @@ -17,7 +17,7 @@ import typing as T from . import ExtensionModule, ModuleReturnValue from .. import mlog -from ..build import BothLibraries, BuildTarget, CustomTargetIndex, Executable, ExtractedObjects, GeneratedList, IncludeDirs, CustomTarget +from ..build import BothLibraries, BuildTarget, CustomTargetIndex, Executable, ExtractedObjects, GeneratedList, IncludeDirs, CustomTarget, StructuredSources from ..dependencies import Dependency, ExternalLibrary from ..interpreter.interpreter import TEST_KWARGS from ..interpreterbase import ContainerTypeInfo, InterpreterException, KwargInfo, FeatureNew, typed_kwargs, typed_pos_args, noPosargs @@ -149,10 +149,9 @@ class RustModule(ExtensionModule): new_target_kwargs['dependencies'] = new_target_kwargs.get('dependencies', []) + dependencies new_target = Executable( - name, base_target.subdir, state.subproject, - base_target.for_machine, base_target.sources, - base_target.objects, base_target.environment, - new_target_kwargs + name, base_target.subdir, state.subproject, base_target.for_machine, + base_target.sources, base_target.structured_sources, + base_target.objects, base_target.environment, new_target_kwargs ) test = self.interpreter.make_test( @@ -203,7 +202,7 @@ class RustModule(ExtensionModule): name: str if isinstance(header, File): name = header.fname - elif isinstance(header, (BuildTarget, BothLibraries, ExtractedObjects)): + elif isinstance(header, (BuildTarget, BothLibraries, ExtractedObjects, StructuredSources)): raise InterpreterException('bindgen source file must be a C header, not an object or build target') else: name = header.get_outputs()[0] diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 665e0c62d..b424b15c4 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -3988,6 +3988,7 @@ class AllPlatformTests(BasePlatformTests): def output_name(name, type_): return type_(name=name, subdir=None, subproject=None, for_machine=MachineChoice.HOST, sources=[], + structured_sources=None, objects=[], environment=env, kwargs={}).filename shared_lib_name = lambda name: output_name(name, SharedLibrary)