From 5dd8171fb3d226eaef52faa821286cf79d0f8a2a Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Sun, 22 Aug 2021 17:16:47 +0200 Subject: [PATCH] docs: Use a custom hotdoc extension for links to RefMan --- docs/extensions/refman_links.py | 108 ++++++++++++++++ docs/meson.build | 9 +- docs/refman/generatorbase.py | 8 +- docs/refman/generatormd.py | 116 +++++++----------- docs/refman/main.py | 3 +- docs/refman/templates/dummy.mustache | 2 +- docs/refman/templates/root.functions.mustache | 2 +- 7 files changed, 171 insertions(+), 77 deletions(-) create mode 100644 docs/extensions/refman_links.py diff --git a/docs/extensions/refman_links.py b/docs/extensions/refman_links.py new file mode 100644 index 000000000..857d2cbcb --- /dev/null +++ b/docs/extensions/refman_links.py @@ -0,0 +1,108 @@ +from pathlib import Path +from json import loads +import re + +from hotdoc.core.exceptions import HotdocSourceException +from hotdoc.core.extension import Extension +from hotdoc.core.tree import Page +from hotdoc.core.project import Project +from hotdoc.run_hotdoc import Application +from hotdoc.core.formatter import Formatter +from hotdoc.utils.loggable import Logger, warn, info + +import typing as T + +if T.TYPE_CHECKING: + import argparse + +Logger.register_warning_code('unknown-refman-link', HotdocSourceException, 'refman-links') + +class RefmanLinksExtension(Extension): + extension_name = 'refman-links' + argument_prefix = 'refman' + + def __init__(self, app: Application, project: Project): + self.project: Project + super().__init__(app, project) + self._data_file: T.Optional[Path] = None + self._data: T.Dict[str, str] = {} + + @staticmethod + def add_arguments(parser: 'argparse.ArgumentParser'): + group = parser.add_argument_group( + 'Refman links', + 'Custom Meson extension', + ) + + # Add Arguments with `group.add_argument(...)` + group.add_argument( + f'--refman-data-file', + help="JSON file with the mappings to replace", + default=None, + ) + + def parse_config(self, config: T.Dict[str, T.Any]) -> None: + super().parse_config(config) + self._data_file = config.get('refman_data_file') + + def _formatting_page_cb(self, formatter: Formatter, page: Page) -> None: + ''' Replace Meson refman tags + + Links of the form [[function]] are automatically replaced + with valid links to the correct URL. To reference objects / types use the + [[@object]] syntax. + ''' + link_regex = re.compile(r'\[\[#?@?([ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\.)*[ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\]\]', re.MULTILINE) + for m in link_regex.finditer(page.formatted_contents): + i = m.group() + obj_id: str = i[2:-2] + obj_id = re.sub(r'[ \n\t]', '', obj_id) # Remove whitespaces + + # Marked as inside a code block? + in_code_block = False + if obj_id.startswith('#'): + in_code_block = True + obj_id = obj_id[1:] + + if obj_id not in self._data: + warn('unknown-refman-link', f'{Path(page.name).name}: Unknown Meson refman link: "{obj_id}"') + continue + + # Just replaces [[!file.id]] paths with the page file (no fancy HTML) + if obj_id.startswith('!'): + page.formatted_contents = page.formatted_contents.replace(i, self._data[obj_id]) + continue + + # Fancy links for functions and methods + text = obj_id + if text.startswith('@'): + text = text[1:] + else: + text = text + '()' + if not in_code_block: + text = f'{text}' + link = f'{text}' + page.formatted_contents = page.formatted_contents.replace(i, link) + + def setup(self) -> None: + super().setup() + + if not self._data_file: + info('Meson refman extension DISABLED') + return + + raw = Path(self._data_file).read_text(encoding='utf-8') + self._data = loads(raw) + + # Register formater + for ext in self.project.extensions.values(): + ext = T.cast(Extension, ext) + ext.formatter.formatting_page_signal.connect(self._formatting_page_cb) + info('Meson refman extension LOADED') + + @staticmethod + def get_dependencies() -> T.List[T.Type[Extension]]: + return [] # In case this extension has dependencies on other extensions + +def get_extension_classes() -> T.List[T.Type[Extension]]: + return [RefmanLinksExtension] diff --git a/docs/meson.build b/docs/meson.build index 9bd80ba7c..fcb4f7f5f 100644 --- a/docs/meson.build +++ b/docs/meson.build @@ -18,13 +18,14 @@ docs_gen = custom_target( refman_gen = custom_target( 'gen_refman', input: files('sitemap.txt'), - output: 'configured_sitemap.txt', + output: ['configured_sitemap.txt', 'refman_links.json'], depfile: 'reman_dep.d', command: [ find_program('./genrefman.py'), '-g', 'md', '-s', '@INPUT@', - '-o', '@OUTPUT@', + '-o', '@OUTPUT0@', + '--link-defs', '@OUTPUT1@', '--depfile', '@DEPFILE@', '--force-color', ], @@ -33,7 +34,7 @@ refman_gen = custom_target( hotdoc = import('hotdoc') documentation = hotdoc.generate_doc(meson.project_name(), project_version: meson.project_version(), - sitemap: refman_gen, + sitemap: refman_gen[0], build_by_default: true, depends: docs_gen, index: 'markdown/index.md', @@ -46,6 +47,8 @@ documentation = hotdoc.generate_doc(meson.project_name(), edit_on_github_repository: 'https://github.com/mesonbuild/meson', syntax_highlighting_activate: true, keep_markup_in_code_blocks: true, + extra_extension: meson.current_source_dir() / 'extensions' / 'refman_links.py', + refman_data_file: refman_gen[1], ) run_target('upload', diff --git a/docs/refman/generatorbase.py b/docs/refman/generatorbase.py index 517c59262..e4041747b 100644 --- a/docs/refman/generatorbase.py +++ b/docs/refman/generatorbase.py @@ -15,7 +15,7 @@ from abc import ABCMeta, abstractmethod import typing as T -from .model import ReferenceManual, Function, Object, ObjectType, NamedObject +from .model import ReferenceManual, Function, Method, Object, ObjectType, NamedObject _N = T.TypeVar('_N', bound=NamedObject) @@ -37,7 +37,11 @@ class GeneratorBase(metaclass=ABCMeta): @staticmethod def sorted_and_filtered(raw: T.List[_N]) -> T.List[_N]: - return sorted([x for x in raw if not x.hidden], key=lambda x: x.name) + def key_fn(fn: Function) -> str: + if isinstance(fn, Method): + return f'1_{fn.obj.name}.{fn.name}' + return f'0_{fn.name}' + return sorted([x for x in raw if not x.hidden], key=key_fn) @property def functions(self) -> T.List[Function]: diff --git a/docs/refman/generatormd.py b/docs/refman/generatormd.py index 831b6816f..6aa0d7873 100644 --- a/docs/refman/generatormd.py +++ b/docs/refman/generatormd.py @@ -14,6 +14,7 @@ from .generatorbase import GeneratorBase import re +import json from .model import ( ReferenceManual, @@ -72,10 +73,11 @@ def code_block(code: str) -> str: return f'
{code}
' class GeneratorMD(GeneratorBase): - def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path) -> None: + def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path, link_def_out: Path) -> None: super().__init__(manual) self.sitemap_out = sitemap_out.resolve() self.sitemap_in = sitemap_in.resolve() + self.link_def_out = link_def_out.resolve() self.out_dir = self.sitemap_out.parent self.generated_files: T.Dict[str, str] = {} @@ -88,29 +90,6 @@ class GeneratorMD(GeneratorBase): parts = [re.sub(r'[0-9]+_', '', x) for x in parts] return f'{"_".join(parts)}.{extension}' - def _object_from_ref(self, ref_str: str) -> T.Union[Function, Object]: - ids = ref_str.split('.') - ids = [x.strip() for x in ids] - assert len(ids) in [1, 2], f'Invalid object id "{ref_str}"' - assert not (ids[0].startswith('@') and len(ids) == 2), f'Invalid object id "{ref_str}"' - if ids[0].startswith('@'): - for obj in self.objects: - if obj.name == ids[0][1:]: - return obj - if len(ids) == 2: - for obj in self.objects: - if obj.name != ids[0]: - continue - for m in obj.methods: - if m.name == ids[1]: - return m - raise RuntimeError(f'Unknown method {ids[1]} in object {ids[0]}') - raise RuntimeError(f'Unknown object {ids[0]}') - for func in self.functions: - if func.name == ids[0]: - return func - raise RuntimeError(f'Unknown function or object {ids[0]}') - def _gen_object_file_id(self, obj: Object) -> str: ''' Deterministically generate a unique file ID for the Object. @@ -122,52 +101,25 @@ class GeneratorMD(GeneratorBase): return f'{base}.{obj.name}' return f'root.{_OBJ_ID_MAP[obj.obj_type]}.{obj.name}' - def _link_to_object(self, obj: T.Union[Function, Object], text: T.Optional[str] = None) -> str: + def _link_to_object(self, obj: T.Union[Function, Object], in_code_block: bool = False) -> str: ''' - Generate a link to the function/method/object documentation. - - The generated link is an HTML link (text) instead of - a Markdown link, so that the generated links can be used in custom - (or rather manual) code blocks. + Generate a palaceholder tag for the the function/method/object documentation. + This tag is then replaced in the custom hotdoc plugin. ''' + prefix = '#' if in_code_block else '' if isinstance(obj, Object): - text = text or f'{obj.name}' - link = self._gen_filename(self._gen_object_file_id(obj), extension="html") + return f'[[{prefix}@{obj.name}]]' elif isinstance(obj, Method): - text = text or f'`{obj.obj.name}.{obj.name}()`' - file = self._gen_filename(self._gen_object_file_id(obj.obj), extension="html") - link = f'{file}#{obj.obj.name}{obj.name}' + return f'[[{prefix}{obj.obj.name}.{obj.name}]]' elif isinstance(obj, Function): - text = text or f'`{obj.name}()`' - link = f'{self._gen_filename("root.functions", extension="html")}#{obj.name}' + return f'[[{prefix}{obj.name}]]' else: raise RuntimeError(f'Invalid argument {obj}') - return f'{text}' def _write_file(self, data: str, file_id: str) -> None:# - ''' Write the data to disk. + ''' Write the data to disk ans store the id for the generated data ''' - Additionally, links of the form [[function]] are automatically replaced - with valid links to the correct URL. To reference objects / types use the - [[@object]] syntax. - - Placeholders with the syntax [[!file_id]] will be replaced with the - corresponding generated markdown file. - ''' self.generated_files[file_id] = self._gen_filename(file_id) - - # Replace [[func_name]] and [[obj.method_name]] with links - link_regex = re.compile(r'\[\[[^\]]+\]\]') - matches = link_regex.findall(data) - for i in matches: - obj_id: str = i[2:-2] - if obj_id.startswith('!'): - link_file_id = obj_id[1:] - data = data.replace(i, self._gen_filename(link_file_id)) - else: - obj = self._object_from_ref(obj_id) - data = data.replace(i, self._link_to_object(obj)) - out_file = self.out_dir / self.generated_files[file_id] out_file.write_text(data, encoding='ascii') mlog.log('Generated', mlog.bold(out_file.name)) @@ -193,11 +145,11 @@ class GeneratorMD(GeneratorBase): # Actual generator functions def _gen_func_or_method(self, func: Function) -> FunctionDictType: - def render_type(typ: Type) -> str: + def render_type(typ: Type, in_code_block: bool = False) -> str: def data_type_to_str(dt: DataTypeInfo) -> str: - base = self._link_to_object(dt.data_type, f'{dt.data_type.name}') + base = self._link_to_object(dt.data_type, in_code_block) if dt.holds: - return f'{base}[{render_type(dt.holds)}]' + return f'{base}[{render_type(dt.holds, in_code_block)}]' return base assert typ.resolved return ' | '.join([data_type_to_str(x) for x in typ.resolved]) @@ -208,11 +160,11 @@ class GeneratorMD(GeneratorBase): def render_signature() -> str: # Skip a lot of computations if the function does not take any arguments if not any([func.posargs, func.optargs, func.kwargs, func.varargs]): - return f'{render_type(func.returns)} {func.name}()' + return f'{render_type(func.returns, True)} {func.name}()' signature = dedent(f'''\ # {self.brief(func)} - {render_type(func.returns)} {func.name}( + {render_type(func.returns, True)} {func.name}( ''') # Calculate maximum lengths of the type and name @@ -229,7 +181,7 @@ class GeneratorMD(GeneratorBase): # Generate some common strings def prepare(arg: ArgBase) -> T.Tuple[str, str, str, str]: - type_str = render_type(arg.type) + type_str = render_type(arg.type, True) type_len = len_stripped(type_str) type_space = ' ' * (max_type_len - type_len) name_space = ' ' * (max_name_len - len(arg.name)) @@ -362,6 +314,7 @@ class GeneratorMD(GeneratorBase): return ret data = { + 'root': self._gen_filename('root'), 'elementary': gen_obj_links(self.elementary), 'returned': gen_obj_links(self.returned), 'builtins': gen_obj_links(self.builtins), @@ -369,11 +322,13 @@ class GeneratorMD(GeneratorBase): 'functions': [{'indent': '', 'link': self._link_to_object(x), 'brief': self.brief(x)} for x in self.functions], } + dummy = {'root': self._gen_filename('root')} + self._write_template(data, 'root') - self._write_template({'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy') - self._write_template({'name': 'Builtin objects'}, f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}', 'dummy') - self._write_template({'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}', 'dummy') - self._write_template({'name': 'Modules'}, f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}', 'dummy') + self._write_template({**dummy, 'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy') + self._write_template({**dummy, 'name': 'Builtin objects'}, f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}', 'dummy') + self._write_template({**dummy, 'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}', 'dummy') + self._write_template({**dummy, 'name': 'Modules'}, f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}', 'dummy') def generate(self) -> None: @@ -384,6 +339,7 @@ class GeneratorMD(GeneratorBase): self._write_object(obj) self._root_refman_docs() self._configure_sitemap() + self._generate_link_def() def _configure_sitemap(self) -> None: ''' @@ -403,3 +359,25 @@ class GeneratorMD(GeneratorBase): indent = base_indent + '\t' * k.count('.') out += f'{indent}{self.generated_files[k]}\n' self.sitemap_out.write_text(out, encoding='utf-8') + + def _generate_link_def(self) -> None: + ''' + Generate the link definition file for the refman_links hotdoc + plugin. The plugin is then responsible for replacing the [[tag]] + tags with custom HTML elements. + ''' + data: T.Dict[str, str] = {} + + # Objects and methods + for obj in self.objects: + obj_file = self._gen_filename(self._gen_object_file_id(obj), extension='html') + data[f'@{obj.name}'] = obj_file + for m in obj.methods: + data[f'{obj.name}.{m.name}'] = f'{obj_file}#{obj.name}{m.name}' + + # Functions + funcs_file = self._gen_filename('root.functions', extension='html') + for fn in self.functions: + data[fn.name] = f'{funcs_file}#{fn.name}' + + self.link_def_out.write_text(json.dumps(data, indent=2), encoding='utf-8') diff --git a/docs/refman/main.py b/docs/refman/main.py index f4b307669..cb040cebe 100644 --- a/docs/refman/main.py +++ b/docs/refman/main.py @@ -34,6 +34,7 @@ def main() -> int: parser.add_argument('-g', '--generator', type=str, choices=['print', 'pickle', 'md'], required=True, help='Generator backend') parser.add_argument('-s', '--sitemap', type=Path, default=meson_root / 'docs' / 'sitemap.txt', help='Path to the input sitemap.txt') parser.add_argument('-o', '--out', type=Path, required=True, help='Output directory for generated files') + parser.add_argument('--link-defs', type=Path, help='Output file for the MD generator link definition file') parser.add_argument('--depfile', type=Path, default=None, help='Set to generate a depfile') parser.add_argument('--force-color', action='store_true', help='Force enable colors') args = parser.parse_args() @@ -51,7 +52,7 @@ def main() -> int: generators: T.Dict[str, T.Callable[[], GeneratorBase]] = { 'print': lambda: GeneratorPrint(refMan), 'pickle': lambda: GeneratorPickle(refMan, args.out), - 'md': lambda: GeneratorMD(refMan, args.out, args.sitemap), + 'md': lambda: GeneratorMD(refMan, args.out, args.sitemap, args.link_defs), } generator = generators[args.generator]() diff --git a/docs/refman/templates/dummy.mustache b/docs/refman/templates/dummy.mustache index b6dc35202..ddb090e44 100644 --- a/docs/refman/templates/dummy.mustache +++ b/docs/refman/templates/dummy.mustache @@ -4,5 +4,5 @@ render-subpages: true ... # {{name}} -See the [root manual document]([[!root]]) for +See the [root manual document]({{root}}) for a general overview. diff --git a/docs/refman/templates/root.functions.mustache b/docs/refman/templates/root.functions.mustache index aa0230db2..33fba5952 100644 --- a/docs/refman/templates/root.functions.mustache +++ b/docs/refman/templates/root.functions.mustache @@ -6,7 +6,7 @@ render-subpages: false # Functions This document lists all functions available in `meson.build` files. -See the [root manual document]([[!root]]) for +See the [root manual document]({{root}}) for an overview of all features. {{#functions}}