docs: Use a custom hotdoc extension for links to RefMan

docs/extensions/refman_links.py

@@ -0,0 +1,108 @@
+from pathlib import Path
+from json import loads
+import re
+
+from hotdoc.core.exceptions import HotdocSourceException
+from hotdoc.core.extension import Extension
+from hotdoc.core.tree import Page
+from hotdoc.core.project import Project
+from hotdoc.run_hotdoc import Application
+from hotdoc.core.formatter import Formatter
+from hotdoc.utils.loggable import Logger, warn, info
+
+import typing as T
+
+if T.TYPE_CHECKING:
+    import argparse
+
+Logger.register_warning_code('unknown-refman-link', HotdocSourceException, 'refman-links')
+
+class RefmanLinksExtension(Extension):
+    extension_name = 'refman-links'
+    argument_prefix = 'refman'
+
+    def __init__(self, app: Application, project: Project):
+        self.project: Project
+        super().__init__(app, project)
+        self._data_file: T.Optional[Path] = None
+        self._data: T.Dict[str, str] = {}
+
+    @staticmethod
+    def add_arguments(parser: 'argparse.ArgumentParser'):
+        group = parser.add_argument_group(
+            'Refman links',
+            'Custom Meson extension',
+        )
+
+        # Add Arguments with `group.add_argument(...)`
+        group.add_argument(
+            f'--refman-data-file',
+            help="JSON file with the mappings to replace",
+            default=None,
+        )
+
+    def parse_config(self, config: T.Dict[str, T.Any]) -> None:
+        super().parse_config(config)
+        self._data_file = config.get('refman_data_file')
+
+    def _formatting_page_cb(self, formatter: Formatter, page: Page) -> None:
+        ''' Replace Meson refman tags
+
+        Links of the form [[function]] are automatically replaced
+        with valid links to the correct URL. To reference objects / types use the
+        [[@object]] syntax.
+        '''
+        link_regex = re.compile(r'\[\[#?@?([ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\.)*[ \n\t]*[a-zA-Z0-9_]+[ \n\t]*\]\]', re.MULTILINE)
+        for m in link_regex.finditer(page.formatted_contents):
+            i = m.group()
+            obj_id: str = i[2:-2]
+            obj_id = re.sub(r'[ \n\t]', '', obj_id)  # Remove whitespaces
+
+            # Marked as inside a code block?
+            in_code_block = False
+            if obj_id.startswith('#'):
+                in_code_block = True
+                obj_id = obj_id[1:]
+
+            if obj_id not in self._data:
+                warn('unknown-refman-link', f'{Path(page.name).name}: Unknown Meson refman link: "{obj_id}"')
+                continue
+
+            # Just replaces [[!file.id]] paths with the page file (no fancy HTML)
+            if obj_id.startswith('!'):
+                page.formatted_contents = page.formatted_contents.replace(i, self._data[obj_id])
+                continue
+
+            # Fancy links for functions and methods
+            text = obj_id
+            if text.startswith('@'):
+                text = text[1:]
+            else:
+                text = text + '()'
+            if not in_code_block:
+                text = f'<code>{text}</code>'
+            link = f'<a href="{self._data[obj_id]}"><ins>{text}</ins></a>'
+            page.formatted_contents = page.formatted_contents.replace(i, link)
+
+    def setup(self) -> None:
+        super().setup()
+
+        if not self._data_file:
+            info('Meson refman extension DISABLED')
+            return
+
+        raw = Path(self._data_file).read_text(encoding='utf-8')
+        self._data = loads(raw)
+
+        # Register formater
+        for ext in self.project.extensions.values():
+            ext = T.cast(Extension, ext)
+            ext.formatter.formatting_page_signal.connect(self._formatting_page_cb)
+        info('Meson refman extension LOADED')
+
+    @staticmethod
+    def get_dependencies() -> T.List[T.Type[Extension]]:
+        return []  # In case this extension has dependencies on other extensions
+
+def get_extension_classes() -> T.List[T.Type[Extension]]:
+    return [RefmanLinksExtension]

docs/meson.build

@@ -18,13 +18,14 @@ docs_gen = custom_target(
 refman_gen = custom_target(
     'gen_refman',
     input: files('sitemap.txt'),
-    output: 'configured_sitemap.txt',
+    output: ['configured_sitemap.txt', 'refman_links.json'],
     depfile: 'reman_dep.d',
     command: [
         find_program('./genrefman.py'),
         '-g', 'md',
         '-s', '@INPUT@',
-        '-o', '@OUTPUT@',
+        '-o', '@OUTPUT0@',
+        '--link-defs', '@OUTPUT1@',
         '--depfile', '@DEPFILE@',
         '--force-color',
     ],
@@ -33,7 +34,7 @@ refman_gen = custom_target(
 hotdoc = import('hotdoc')
 documentation = hotdoc.generate_doc(meson.project_name(),
     project_version: meson.project_version(),
-    sitemap: refman_gen,
+    sitemap: refman_gen[0],
     build_by_default: true,
     depends: docs_gen,
     index: 'markdown/index.md',
@@ -46,6 +47,8 @@ documentation = hotdoc.generate_doc(meson.project_name(),
     edit_on_github_repository: 'https://github.com/mesonbuild/meson',
     syntax_highlighting_activate: true,
     keep_markup_in_code_blocks: true,
+    extra_extension: meson.current_source_dir() / 'extensions' / 'refman_links.py',
+    refman_data_file: refman_gen[1],
 )
 
 run_target('upload',

docs/refman/generatorbase.py

@@ -15,7 +15,7 @@
 from abc import ABCMeta, abstractmethod
 import typing as T
 
-from .model import ReferenceManual, Function, Object, ObjectType, NamedObject
+from .model import ReferenceManual, Function, Method, Object, ObjectType, NamedObject
 
 _N = T.TypeVar('_N', bound=NamedObject)
 
@@ -37,7 +37,11 @@ class GeneratorBase(metaclass=ABCMeta):
 
     @staticmethod
     def sorted_and_filtered(raw: T.List[_N]) -> T.List[_N]:
-        return sorted([x for x in raw if not x.hidden], key=lambda x: x.name)
+        def key_fn(fn: Function) -> str:
+            if isinstance(fn, Method):
+                return f'1_{fn.obj.name}.{fn.name}'
+            return f'0_{fn.name}'
+        return sorted([x for x in raw if not x.hidden], key=key_fn)
 
     @property
     def functions(self) -> T.List[Function]:

docs/refman/generatormd.py

@@ -14,6 +14,7 @@
 
 from .generatorbase import GeneratorBase
 import re
+import json
 
 from .model import (
     ReferenceManual,
@@ -72,10 +73,11 @@ def code_block(code: str) -> str:
     return f'<pre><code class="language-meson">{code}</code></pre>'
 
 class GeneratorMD(GeneratorBase):
-    def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path) -> None:
+    def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path, link_def_out: Path) -> None:
         super().__init__(manual)
         self.sitemap_out = sitemap_out.resolve()
         self.sitemap_in = sitemap_in.resolve()
+        self.link_def_out = link_def_out.resolve()
         self.out_dir = self.sitemap_out.parent
         self.generated_files: T.Dict[str, str] = {}
 
@@ -88,29 +90,6 @@ class GeneratorMD(GeneratorBase):
         parts = [re.sub(r'[0-9]+_', '', x) for x in parts]
         return f'{"_".join(parts)}.{extension}'
 
-    def _object_from_ref(self, ref_str: str) -> T.Union[Function, Object]:
-        ids = ref_str.split('.')
-        ids = [x.strip() for x in ids]
-        assert len(ids) in [1, 2], f'Invalid object id "{ref_str}"'
-        assert not (ids[0].startswith('@') and len(ids) == 2), f'Invalid object id "{ref_str}"'
-        if ids[0].startswith('@'):
-            for obj in self.objects:
-                if obj.name == ids[0][1:]:
-                    return obj
-        if len(ids) == 2:
-            for obj in self.objects:
-                if obj.name != ids[0]:
-                    continue
-                for m in obj.methods:
-                    if m.name == ids[1]:
-                        return m
-                raise RuntimeError(f'Unknown method {ids[1]} in object {ids[0]}')
-            raise RuntimeError(f'Unknown object {ids[0]}')
-        for func in self.functions:
-            if func.name == ids[0]:
-                return func
-        raise RuntimeError(f'Unknown function or object {ids[0]}')
-
     def _gen_object_file_id(self, obj: Object) -> str:
         '''
             Deterministically generate a unique file ID for the Object.
@@ -122,52 +101,25 @@ class GeneratorMD(GeneratorBase):
             return f'{base}.{obj.name}'
         return f'root.{_OBJ_ID_MAP[obj.obj_type]}.{obj.name}'
 
-    def _link_to_object(self, obj: T.Union[Function, Object], text: T.Optional[str] = None) -> str:
+    def _link_to_object(self, obj: T.Union[Function, Object], in_code_block: bool = False) -> str:
         '''
-            Generate a link to the function/method/object documentation.
+            Generate a palaceholder tag for the the function/method/object documentation.
-
+            This tag is then replaced in the custom hotdoc plugin.
-            The generated link is an HTML link (<a href="">text</a>) instead of
-            a Markdown link, so that the generated links can be used in custom
-            (or rather manual) code blocks.
         '''
+        prefix = '#' if in_code_block else ''
         if isinstance(obj, Object):
-            text = text or f'<ins><code>{obj.name}</code></ins>'
+            return f'[[{prefix}@{obj.name}]]'
-            link = self._gen_filename(self._gen_object_file_id(obj), extension="html")
         elif isinstance(obj, Method):
-            text = text or f'<ins><code>`{obj.obj.name}.{obj.name}()`</code></ins>'
+            return f'[[{prefix}{obj.obj.name}.{obj.name}]]'
-            file = self._gen_filename(self._gen_object_file_id(obj.obj), extension="html")
-            link = f'{file}#{obj.obj.name}{obj.name}'
         elif isinstance(obj, Function):
-            text = text or f'<ins><code>`{obj.name}()`</code></ins>'
+            return f'[[{prefix}{obj.name}]]'
-            link = f'{self._gen_filename("root.functions", extension="html")}#{obj.name}'
         else:
             raise RuntimeError(f'Invalid argument {obj}')
-        return f'<a href="{link}">{text}</a>'
 
     def _write_file(self, data: str, file_id: str) -> None:#
-        ''' Write the data to disk.
+        ''' Write the data to disk ans store the id for the generated data '''
 
-        Additionally, links of the form [[function]] are automatically replaced
-        with valid links to the correct URL. To reference objects / types use the
-        [[@object]] syntax.
-
-        Placeholders with the syntax [[!file_id]] will be replaced with the
-        corresponding generated markdown file.
-        '''
         self.generated_files[file_id] = self._gen_filename(file_id)
-
-        # Replace [[func_name]] and [[obj.method_name]] with links
-        link_regex = re.compile(r'\[\[[^\]]+\]\]')
-        matches = link_regex.findall(data)
-        for i in matches:
-            obj_id: str = i[2:-2]
-            if obj_id.startswith('!'):
-                link_file_id = obj_id[1:]
-                data = data.replace(i, self._gen_filename(link_file_id))
-            else:
-                obj = self._object_from_ref(obj_id)
-                data = data.replace(i, self._link_to_object(obj))
-
         out_file = self.out_dir / self.generated_files[file_id]
         out_file.write_text(data, encoding='ascii')
         mlog.log('Generated', mlog.bold(out_file.name))
@@ -193,11 +145,11 @@ class GeneratorMD(GeneratorBase):
 
     # Actual generator functions
     def _gen_func_or_method(self, func: Function) -> FunctionDictType:
-        def render_type(typ: Type) -> str:
+        def render_type(typ: Type, in_code_block: bool = False) -> str:
             def data_type_to_str(dt: DataTypeInfo) -> str:
-                base = self._link_to_object(dt.data_type, f'<ins>{dt.data_type.name}</ins>')
+                base = self._link_to_object(dt.data_type, in_code_block)
                 if dt.holds:
-                    return f'{base}[{render_type(dt.holds)}]'
+                    return f'{base}[{render_type(dt.holds, in_code_block)}]'
                 return base
             assert typ.resolved
             return ' | '.join([data_type_to_str(x) for x in typ.resolved])
@@ -208,11 +160,11 @@ class GeneratorMD(GeneratorBase):
         def render_signature() -> str:
             # Skip a lot of computations if the function does not take any arguments
             if not any([func.posargs, func.optargs, func.kwargs, func.varargs]):
-                return f'{render_type(func.returns)} {func.name}()'
+                return f'{render_type(func.returns, True)} {func.name}()'
 
             signature = dedent(f'''\
                 # {self.brief(func)}
-                {render_type(func.returns)} {func.name}(
+                {render_type(func.returns, True)} {func.name}(
             ''')
 
             # Calculate maximum lengths of the type and name
@@ -229,7 +181,7 @@ class GeneratorMD(GeneratorBase):
 
             # Generate some common strings
             def prepare(arg: ArgBase) -> T.Tuple[str, str, str, str]:
-                type_str = render_type(arg.type)
+                type_str = render_type(arg.type, True)
                 type_len = len_stripped(type_str)
                 type_space = ' ' * (max_type_len - type_len)
                 name_space = ' ' * (max_name_len - len(arg.name))
@@ -362,6 +314,7 @@ class GeneratorMD(GeneratorBase):
             return ret
 
         data = {
+            'root': self._gen_filename('root'),
             'elementary': gen_obj_links(self.elementary),
             'returned': gen_obj_links(self.returned),
             'builtins': gen_obj_links(self.builtins),
@@ -369,11 +322,13 @@ class GeneratorMD(GeneratorBase):
             'functions': [{'indent': '', 'link': self._link_to_object(x), 'brief': self.brief(x)} for x in self.functions],
         }
 
+        dummy = {'root': self._gen_filename('root')}
+
         self._write_template(data, 'root')
-        self._write_template({'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy')
+        self._write_template({**dummy, 'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy')
-        self._write_template({'name': 'Builtin objects'},  f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}',    'dummy')
+        self._write_template({**dummy, 'name': 'Builtin objects'},  f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}',    'dummy')
-        self._write_template({'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}',   'dummy')
+        self._write_template({**dummy, 'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}',   'dummy')
-        self._write_template({'name': 'Modules'},          f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}',     'dummy')
+        self._write_template({**dummy, 'name': 'Modules'},          f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}',     'dummy')
 
 
     def generate(self) -> None:
@@ -384,6 +339,7 @@ class GeneratorMD(GeneratorBase):
                 self._write_object(obj)
             self._root_refman_docs()
             self._configure_sitemap()
+            self._generate_link_def()
 
     def _configure_sitemap(self) -> None:
         '''
@@ -403,3 +359,25 @@ class GeneratorMD(GeneratorBase):
                 indent = base_indent + '\t' * k.count('.')
                 out += f'{indent}{self.generated_files[k]}\n'
         self.sitemap_out.write_text(out, encoding='utf-8')
+
+    def _generate_link_def(self) -> None:
+        '''
+            Generate the link definition file for the refman_links hotdoc
+            plugin. The plugin is then responsible for replacing the [[tag]]
+            tags with custom HTML elements.
+        '''
+        data: T.Dict[str, str] = {}
+
+        # Objects and methods
+        for obj in self.objects:
+            obj_file = self._gen_filename(self._gen_object_file_id(obj), extension='html')
+            data[f'@{obj.name}'] = obj_file
+            for m in obj.methods:
+                data[f'{obj.name}.{m.name}'] = f'{obj_file}#{obj.name}{m.name}'
+
+        # Functions
+        funcs_file = self._gen_filename('root.functions', extension='html')
+        for fn in self.functions:
+            data[fn.name] = f'{funcs_file}#{fn.name}'
+
+        self.link_def_out.write_text(json.dumps(data, indent=2), encoding='utf-8')

docs/refman/main.py

@@ -34,6 +34,7 @@ def main() -> int:
     parser.add_argument('-g', '--generator', type=str, choices=['print', 'pickle', 'md'], required=True, help='Generator backend')
     parser.add_argument('-s', '--sitemap', type=Path, default=meson_root / 'docs' / 'sitemap.txt', help='Path to the input sitemap.txt')
     parser.add_argument('-o', '--out', type=Path, required=True, help='Output directory for generated files')
+    parser.add_argument('--link-defs', type=Path, help='Output file for the MD generator link definition file')
     parser.add_argument('--depfile', type=Path, default=None, help='Set to generate a depfile')
     parser.add_argument('--force-color', action='store_true', help='Force enable colors')
     args = parser.parse_args()
@@ -51,7 +52,7 @@ def main() -> int:
     generators: T.Dict[str, T.Callable[[], GeneratorBase]] = {
         'print': lambda: GeneratorPrint(refMan),
         'pickle': lambda: GeneratorPickle(refMan, args.out),
-        'md': lambda: GeneratorMD(refMan, args.out, args.sitemap),
+        'md': lambda: GeneratorMD(refMan, args.out, args.sitemap, args.link_defs),
     }
     generator = generators[args.generator]()
 

docs/refman/templates/dummy.mustache

@@ -4,5 +4,5 @@ render-subpages: true
 ...
 # {{name}}
 
-See the [root manual document]([[!root]]) for
+See the [root manual document]({{root}}) for
 a general overview.

docs/refman/templates/root.functions.mustache

@@ -6,7 +6,7 @@ render-subpages: false
 # Functions
 
 This document lists all functions available in `meson.build` files.
-See the [root manual document]([[!root]]) for
+See the [root manual document]({{root}}) for
 an overview of all features.
 
 {{#functions}}