From 5eb55075baa2883170a3d0cf3c0621aae56a1632 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 30 Mar 2023 11:32:44 -0400 Subject: [PATCH] intro: add more details to generated json files This will help with the writing of tools to generate VisualStudio project and solution files, and possibly for other IDEs as well. - Used compilers a about `host`, `build` and `target` machines arere listed in `intro-compilers.json` - Informations lister in `intro-machines.json` - `intro-dependencies.json` now includes internal dependencies, and relations between dependencies. - `intro-targets.json` now includes dependencies, `vs_module_defs`, `win_subsystem`, and linker parameters. --- docs/markdown/snippets/more_intro_data.md | 9 +++ mesonbuild/backend/ninjabackend.py | 61 ++++++++++----- mesonbuild/compilers/compilers.py | 2 +- mesonbuild/dependencies/base.py | 2 +- mesonbuild/mintro.py | 94 ++++++++++++++++++++--- unittests/allplatformstests.py | 42 ++++++++-- 6 files changed, 172 insertions(+), 38 deletions(-) create mode 100644 docs/markdown/snippets/more_intro_data.md diff --git a/docs/markdown/snippets/more_intro_data.md b/docs/markdown/snippets/more_intro_data.md new file mode 100644 index 000000000..2ce65a378 --- /dev/null +++ b/docs/markdown/snippets/more_intro_data.md @@ -0,0 +1,9 @@ +## More data in introspection files + +- Used compilers are listed in `intro-compilers.json` +- Informations about `host`, `build` and `target` machines + are lister in `intro-machines.json` +- `intro-dependencies.json` now includes internal dependencies, + and relations between dependencies. +- `intro-targets.json` now includes dependencies, `vs_module_defs`, + `win_subsystem`, and linker parameters. diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 81be3a157..4d367ed29 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -746,7 +746,8 @@ class NinjaBackend(backends.Backend): return False return True - def create_target_source_introspection(self, target: build.Target, comp: compilers.Compiler, parameters, sources, generated_sources): + def create_target_source_introspection(self, target: build.Target, comp: compilers.Compiler, parameters, sources, generated_sources, + unity_sources: T.Optional[T.List[mesonlib.FileOrString]] = None): ''' Adds the source file introspection information for a language of a target @@ -781,16 +782,40 @@ class NinjaBackend(backends.Backend): 'parameters': parameters, 'sources': [], 'generated_sources': [], + 'unity_sources': [], } tgt[id_hash] = src_block - # Make source files absolute - sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x)) - for x in sources] - generated_sources = [x.absolute_path(self.source_dir, self.build_dir) if isinstance(x, File) else os.path.normpath(os.path.join(self.build_dir, x)) - for x in generated_sources] - # Add the source files - src_block['sources'] += sources - src_block['generated_sources'] += generated_sources + + def compute_path(file: mesonlib.FileOrString) -> str: + """ Make source files absolute """ + if isinstance(file, File): + return file.absolute_path(self.source_dir, self.build_dir) + return os.path.normpath(os.path.join(self.build_dir, file)) + + src_block['sources'].extend(compute_path(x) for x in sources) + src_block['generated_sources'].extend(compute_path(x) for x in generated_sources) + if unity_sources: + src_block['unity_sources'].extend(compute_path(x) for x in unity_sources) + + def create_target_linker_introspection(self, target: build.Target, linker: T.Union[Compiler, StaticLinker], parameters): + tid = target.get_id() + tgt = self.introspection_data[tid] + lnk_hash = tuple(parameters) + lnk_block = tgt.get(lnk_hash, None) + if lnk_block is None: + if isinstance(parameters, CompilerArgs): + parameters = parameters.to_native(copy=True) + + if isinstance(linker, Compiler): + linkers = linker.get_linker_exelist() + else: + linkers = linker.get_exelist() + + lnk_block = { + 'linker': linkers, + 'parameters': parameters, + } + tgt[lnk_hash] = lnk_block def generate_target(self, target): try: @@ -985,7 +1010,7 @@ class NinjaBackend(backends.Backend): if is_unity: for src in self.generate_unity_files(target, unity_src): o, s = self.generate_single_compile(target, src, True, unity_deps + header_deps + d_generated_deps, - fortran_order_deps, fortran_inc_args) + fortran_order_deps, fortran_inc_args, unity_src) obj_list.append(o) compiled_sources.append(s) source2object[s] = o @@ -2809,7 +2834,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) def generate_single_compile(self, target: build.BuildTarget, src, is_generated=False, header_deps=None, order_deps: T.Optional[T.List[str]] = None, - extra_args: T.Optional[T.List[str]] = None) -> None: + extra_args: T.Optional[T.List[str]] = None, + unity_sources: T.Optional[T.List[mesonlib.FileOrString]] = None) -> None: """ Compiles C/C++, ObjC/ObjC++, Fortran, and D sources """ @@ -2832,9 +2858,9 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) # Create introspection information if is_generated is False: - self.create_target_source_introspection(target, compiler, commands, [src], []) + self.create_target_source_introspection(target, compiler, commands, [src], [], unity_sources) else: - self.create_target_source_introspection(target, compiler, commands, [], [src]) + self.create_target_source_introspection(target, compiler, commands, [], [src], unity_sources) build_dir = self.environment.get_build_dir() if isinstance(src, File): @@ -3360,6 +3386,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) elem = NinjaBuildElement(self.all_outputs, outname, linker_rule, obj_list, implicit_outs=implicit_outs) elem.add_dep(dep_targets + custom_target_libraries) elem.add_item('LINK_ARGS', commands) + self.create_target_linker_introspection(target, linker, commands) return elem def get_dependency_filename(self, t): @@ -3555,13 +3582,11 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) self.add_build(elem) def get_introspection_data(self, target_id: str, target: build.Target) -> T.List[T.Dict[str, T.Union[bool, str, T.List[T.Union[str, T.Dict[str, T.Union[str, T.List[str], bool]]]]]]]: - if target_id not in self.introspection_data or len(self.introspection_data[target_id]) == 0: + data = self.introspection_data.get(target_id) + if not data: return super().get_introspection_data(target_id, target) - result = [] - for i in self.introspection_data[target_id].values(): - result += [i] - return result + return list(data.values()) def _scan_fortran_file_deps(src: Path, srcdir: Path, dirname: Path, tdeps, compiler) -> T.List[str]: diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 5f7bfa44b..2f3086acc 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -600,7 +600,7 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta): return self.exelist.copy() if ccache else self.exelist_no_ccache.copy() def get_linker_exelist(self) -> T.List[str]: - return self.linker.get_exelist() + return self.linker.get_exelist() if self.linker else self.get_exelist() @abc.abstractmethod def get_output_args(self, outputname: str) -> T.List[str]: diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index 2cae8c324..2f69b85ce 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -99,7 +99,7 @@ class Dependency(HoldableObject): return kwargs['include_type'] def __init__(self, type_name: DependencyTypeName, kwargs: T.Dict[str, T.Any]) -> None: - self.name = "null" + self.name = f'dep{id(self)}' self.version: T.Optional[str] = None self.language: T.Optional[str] = None # None means C-like self.is_found = False diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 184231317..dd1a00054 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -22,6 +22,7 @@ project files and don't need this info.""" from contextlib import redirect_stdout import collections +import dataclasses import json import os from pathlib import Path, PurePath @@ -31,6 +32,9 @@ import typing as T from . import build, mesonlib, coredata as cdata from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionLevel, AstIDGenerator, AstIndentationGenerator, AstJSONPrinter from .backend import backends +from .dependencies import Dependency +from . import environment +from .interpreterbase import ObjectHolder from .mesonlib import OptionKey from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode @@ -76,10 +80,12 @@ def get_meson_introspection_types(coredata: T.Optional[cdata.CoreData] = None, ('benchmarks', IntroCommand('List all benchmarks', func=lambda: list_benchmarks(benchmarkdata))), ('buildoptions', IntroCommand('List all build options', func=lambda: list_buildoptions(coredata), no_bd=list_buildoptions_from_source)), ('buildsystem_files', IntroCommand('List files that make up the build system', func=lambda: list_buildsystem_files(builddata, interpreter))), - ('dependencies', IntroCommand('List external dependencies', func=lambda: list_deps(coredata), no_bd=list_deps_from_source)), + ('compilers', IntroCommand('List used compilers', func=lambda: list_compilers(coredata))), + ('dependencies', IntroCommand('List external dependencies', func=lambda: list_deps(coredata, backend), no_bd=list_deps_from_source)), ('scan_dependencies', IntroCommand('Scan for dependencies used in the meson.build file', no_bd=list_deps_from_source)), ('installed', IntroCommand('List all installed files and directories', func=lambda: list_installed(installdata))), ('install_plan', IntroCommand('List all installed files and directories with their details', func=lambda: list_install_plan(installdata))), + ('machines', IntroCommand('Information about host, build, and target machines', func=lambda: list_machines(builddata))), ('projectinfo', IntroCommand('Information about projects', func=lambda: list_projinfo(builddata), no_bd=list_projinfo_from_source)), ('targets', IntroCommand('List top level targets', func=lambda: list_targets(builddata, installdata, backend), no_bd=list_targets_from_source)), ('tests', IntroCommand('List all unit tests', func=lambda: list_tests(testdata))), @@ -250,14 +256,22 @@ def list_targets(builddata: build.Build, installdata: backends.InstallData, back 'name': target.get_basename(), 'id': idname, 'type': target.get_typename(), - 'defined_in': os.path.normpath(os.path.join(src_dir, target.subdir, 'meson.build')), + 'defined_in': os.path.normpath(os.path.join(src_dir, target.subdir, environment.build_filename)), 'filename': [os.path.join(build_dir, outdir, x) for x in target.get_outputs()], 'build_by_default': target.build_by_default, 'target_sources': backend.get_introspection_data(idname, target), 'extra_files': [os.path.normpath(os.path.join(src_dir, x.subdir, x.fname)) for x in target.extra_files], - 'subproject': target.subproject or None + 'subproject': target.subproject or None, + 'dependencies': [d.name for d in getattr(target, 'external_deps', [])], } + vs_module_defs = getattr(target, 'vs_module_defs', None) + if vs_module_defs is not None: + t['vs_module_defs'] = vs_module_defs.relative_name() + win_subsystem = getattr(target, 'win_subsystem', None) + if win_subsystem is not None: + t['win_subsystem'] = win_subsystem + if installdata and target.should_install(): t['installed'] = True ifn = [install_lookuptable.get(x, [None]) for x in target.get_outputs()] @@ -343,6 +357,23 @@ def list_buildsystem_files(builddata: build.Build, interpreter: Interpreter) -> filelist = [PurePath(src_dir, x).as_posix() for x in filelist] return filelist +def list_compilers(coredata: cdata.CoreData) -> T.Dict[str, T.Dict[str, T.Dict[str, str]]]: + compilers: T.Dict[str, T.Dict[str, T.Dict[str, str]]] = {} + for machine in ('host', 'build'): + compilers[machine] = {} + for language, compiler in getattr(coredata.compilers, machine).items(): + compilers[machine][language] = { + 'id': compiler.get_id(), + 'exelist': compiler.get_exelist(), + 'linker_exelist': compiler.get_linker_exelist(), + 'file_suffixes': compiler.file_suffixes, + 'default_suffix': compiler.get_default_suffix(), + 'version': compiler.version, + 'full_version': compiler.full_version, + 'linker_id': compiler.get_linker_id(), + } + return compilers + def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, T.Union[str, bool]]]: result = [] # type: T.List[T.Dict[str, T.Union[str, bool]]] for i in intr.dependencies: @@ -356,15 +387,48 @@ def list_deps_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[str, result += [{k: v for k, v in i.items() if k in keys}] return result -def list_deps(coredata: cdata.CoreData) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]: - result = [] # type: T.List[T.Dict[str, T.Union[str, T.List[str]]]] +def list_deps(coredata: cdata.CoreData, backend: backends.Backend) -> T.List[T.Dict[str, T.Union[str, T.List[str]]]]: + result: T.Dict[str, T.Dict[str, T.Union[str, T.List[str]]]] = {} + + def _src_to_str(src_file: T.Union[mesonlib.FileOrString, build.CustomTarget, build.StructuredSources, build.CustomTargetIndex, build.GeneratedList]) -> T.List[str]: + if isinstance(src_file, str): + return [src_file] + if isinstance(src_file, mesonlib.File): + return [src_file.absolute_path(backend.source_dir, backend.build_dir)] + if isinstance(src_file, (build.CustomTarget, build.CustomTargetIndex, build.GeneratedList)): + return src_file.get_outputs() + if isinstance(src_file, build.StructuredSources): + return [f for s in src_file.as_list() for f in _src_to_str(s)] + raise mesonlib.MesonBugException(f'Invalid file type {type(src_file)}.') + + def _create_result(d: Dependency, varname: T.Optional[str] = None) -> T.Dict[str, T.Any]: + return { + 'name': d.name, + 'type': d.type_name, + 'version': d.get_version(), + 'compile_args': d.get_compile_args(), + 'link_args': d.get_link_args(), + 'include_directories': [i for idirs in d.get_include_dirs() for i in idirs.to_string_list(backend.source_dir)], + 'sources': [f for s in d.get_sources() for f in _src_to_str(s)], + 'extra_files': [f for s in d.get_extra_files() for f in _src_to_str(s)], + 'deps': [e.name for e in d.ext_deps], + 'meson_variables': [varname] if varname else [], + } + for d in coredata.deps.host.values(): if d.found(): - result += [{'name': d.name, - 'version': d.get_version(), - 'compile_args': d.get_compile_args(), - 'link_args': d.get_link_args()}] - return result + result[d.name] = _create_result(d) + + for varname, holder in backend.interpreter.variables.items(): + if isinstance(holder, ObjectHolder): + d = holder.held_object + if isinstance(d, Dependency) and d.found(): + if d.name in result: + T.cast(T.List[str], result[d.name]['meson_variables']).append(varname) + else: + result[d.name] = _create_result(d, varname) + + return list(result.values()) def get_test_list(testdata: T.List[backends.TestSerialisation]) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]: result = [] # type: T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]] @@ -396,6 +460,16 @@ def list_tests(testdata: T.List[backends.TestSerialisation]) -> T.List[T.Dict[st def list_benchmarks(benchdata: T.List[backends.TestSerialisation]) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], T.Dict[str, str]]]]: return get_test_list(benchdata) +def list_machines(builddata: build.Build) -> T.Dict[str, T.Dict[str, T.Union[str, bool]]]: + machines: T.Dict[str, T.Dict[str, T.Union[str, bool]]] = {} + for m in ('host', 'build', 'target'): + machine = getattr(builddata.environment.machines, m) + machines[m] = dataclasses.asdict(machine) + machines[m]['is_64_bit'] = machine.is_64_bit + machines[m]['exe_suffix'] = machine.get_exe_suffix() + machines[m]['object_suffix'] = machine.get_object_suffix() + return machines + def list_projinfo(builddata: build.Build) -> T.Dict[str, T.Union[str, T.List[T.Dict[str, str]]]]: result = {'version': builddata.project_version, 'descriptive_name': builddata.project_name, diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 493cb9c03..d6ff3b0ad 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -852,7 +852,7 @@ class AllPlatformTests(BasePlatformTests): name = None for target in targets: for target_sources in target["target_sources"]: - for generated_source in target_sources["generated_sources"]: + for generated_source in target_sources.get("generated_sources", []): if "includestuff.pyx.c" in generated_source: name = generated_source break @@ -1266,6 +1266,8 @@ class AllPlatformTests(BasePlatformTests): # This assumes all of the targets support lto for t in targets: for s in t['target_sources']: + if 'linker' in s: + continue for e in expected: self.assertIn(e, s['parameters']) @@ -2984,8 +2986,11 @@ class AllPlatformTests(BasePlatformTests): ('benchmarks', list), ('buildoptions', list), ('buildsystem_files', list), + ('compilers', dict), ('dependencies', list), + ('install_plan', dict), ('installed', dict), + ('machines', dict), ('projectinfo', dict), ('targets', list), ('tests', list), @@ -3027,9 +3032,15 @@ class AllPlatformTests(BasePlatformTests): dependencies_typelist = [ ('name', str), + ('type', str), ('version', str), ('compile_args', list), ('link_args', list), + ('include_directories', list), + ('sources', list), + ('extra_files', list), + ('deps', list), + ('meson_variables', list), ] targets_typelist = [ @@ -3042,8 +3053,11 @@ class AllPlatformTests(BasePlatformTests): ('target_sources', list), ('extra_files', list), ('subproject', (str, None)), + ('dependencies', list), ('install_filename', (list, None)), ('installed', bool), + ('vs_module_defs', (str, None)), + ('win_subsystem', (str, None)), ] targets_sources_typelist = [ @@ -3052,6 +3066,12 @@ class AllPlatformTests(BasePlatformTests): ('parameters', list), ('sources', list), ('generated_sources', list), + ('unity_sources', (list, None)), + ] + + target_sources_linker_typelist = [ + ('linker', list), + ('parameters', list), ] # First load all files @@ -3075,7 +3095,7 @@ class AllPlatformTests(BasePlatformTests): name_to_out.update({i['name']: i['filename']}) for group in i['target_sources']: src_to_id.update({os.path.relpath(src, testdir): i['id'] - for src in group['sources']}) + for src in group.get('sources', [])}) # Check Tests and benchmarks tests_to_find = ['test case 1', 'test case 2', 'benchmark 1'] @@ -3155,8 +3175,11 @@ class AllPlatformTests(BasePlatformTests): self.assertPathEqual(i['defined_in'], os.path.join(testdir, tgt[3])) targets_to_find.pop(i['name'], None) for j in i['target_sources']: - assertKeyTypes(targets_sources_typelist, j) - self.assertEqual(j['sources'], [os.path.normpath(f) for f in tgt[4]]) + if 'compiler' in j: + assertKeyTypes(targets_sources_typelist, j) + self.assertEqual(j['sources'], [os.path.normpath(f) for f in tgt[4]]) + else: + assertKeyTypes(target_sources_linker_typelist, j) self.assertDictEqual(targets_to_find, {}) def test_introspect_file_dump_equals_all(self): @@ -3169,9 +3192,11 @@ class AllPlatformTests(BasePlatformTests): 'benchmarks', 'buildoptions', 'buildsystem_files', + 'compilers', 'dependencies', 'installed', 'install_plan', + 'machines', 'projectinfo', 'targets', 'tests', @@ -3244,12 +3269,13 @@ class AllPlatformTests(BasePlatformTests): res_wb = [i for i in res_wb if i['type'] != 'custom'] for i in res_wb: i['filename'] = [os.path.relpath(x, self.builddir) for x in i['filename']] - if 'install_filename' in i: - del i['install_filename'] - + for k in ('install_filename', 'dependencies', 'win_subsystem'): + if k in i: + del i[k] + sources = [] for j in i['target_sources']: - sources += j['sources'] + sources += j.get('sources', []) i['target_sources'] = [{ 'language': 'unknown', 'compiler': [],