docs: Added Markdown generator

pull/8960/head
Daniel Mensinger 4 years ago
parent 955a29a92d
commit 476b93fd74
No known key found for this signature in database
GPG Key ID: 54DD94C131E277D4
  1. 405
      docs/refman/generatormd.py
  2. 3
      docs/refman/main.py
  3. 28
      docs/refman/templates/args.mustache
  4. 8
      docs/refman/templates/dummy.mustache
  5. 55
      docs/refman/templates/func.mustache
  6. 14
      docs/refman/templates/notes.mustache
  7. 62
      docs/refman/templates/object.mustache
  8. 20
      docs/refman/templates/root.functions.mustache
  9. 48
      docs/refman/templates/root.mustache
  10. 1
      docs/refman/templates/root_link.mustache
  11. 18
      docs/refman/templates/taggs.mustache

@ -0,0 +1,405 @@
# Copyright 2021 The Meson development team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .generatorbase import GeneratorBase
import re
from .model import (
ReferenceManual,
Function,
Method,
Object,
ObjectType,
Type,
DataTypeInfo,
ArgBase,
PosArg,
VarArgs,
Kwarg,
)
from pathlib import Path
from textwrap import dedent
import typing as T
from mesonbuild import mlog
PlaceholderTypes = T.Union[None, str, bool]
FunctionDictType = T.Dict[
str,
T.Union[
PlaceholderTypes,
T.Dict[str, PlaceholderTypes],
T.Dict[str, T.Dict[str, PlaceholderTypes]],
T.Dict[str, T.List[T.Dict[str, PlaceholderTypes]]],
T.List[T.Dict[str, PlaceholderTypes]],
]
]
_ROOT_BASENAME = 'RefMan'
_OBJ_ID_MAP = {
ObjectType.ELEMENTARY: 'elementary',
ObjectType.BUILTIN: 'builtin',
ObjectType.MODULE: 'module',
ObjectType.RETURNED: 'returned',
}
# Indent all but the first line with 4*depth spaces.
# This function is designed to be used with `dedent`
# and fstrings where multiline strings are used during
# the string interpolation.
def smart_indent(raw: str, depth: int = 3) -> str:
lines = raw.split('\n')
first_line = lines[0]
lines = [' ' * (4 * depth) + x for x in lines]
lines[0] = first_line # Do not indent the first line
return '\n'.join(lines)
def code_block(code: str) -> str:
code = dedent(code)
return f'<pre><code class="language-meson">{code}</code></pre>'
class GeneratorMD(GeneratorBase):
def __init__(self, manual: ReferenceManual, sitemap_out: Path, sitemap_in: Path) -> None:
super().__init__(manual)
self.sitemap_out = sitemap_out.resolve()
self.sitemap_in = sitemap_in.resolve()
self.out_dir = self.sitemap_out.parent
self.generated_files: T.Dict[str, str] = {}
# Utility functions
def _gen_filename(self, file_id: str, *, extension: str = 'md') -> str:
parts = file_id.split('.')
assert parts[0] == 'root'
assert all([x for x in parts])
parts[0] = _ROOT_BASENAME
parts = [re.sub(r'[0-9]+_', '', x) for x in parts]
return f'{"_".join(parts)}.{extension}'
def _object_from_ref(self, ref_str: str) -> T.Union[Function, Object]:
ids = ref_str.split('.')
ids = [x.strip() for x in ids]
assert len(ids) in [1, 2], f'Invalid object id "{ref_str}"'
assert not (ids[0].startswith('@') and len(ids) == 2), f'Invalid object id "{ref_str}"'
if ids[0].startswith('@'):
for obj in self.objects:
if obj.name == ids[0][1:]:
return obj
if len(ids) == 2:
for obj in self.objects:
if obj.name != ids[0]:
continue
for m in obj.methods:
if m.name == ids[1]:
return m
raise RuntimeError(f'Unknown method {ids[1]} in object {ids[0]}')
raise RuntimeError(f'Unknown object {ids[0]}')
for func in self.functions:
if func.name == ids[0]:
return func
raise RuntimeError(f'Unknown function or object {ids[0]}')
def _gen_object_file_id(self, obj: Object) -> str:
'''
Deterministically generate a unique file ID for the Object.
This ID determines where the object will be inserted in the sitemap.
'''
if obj.obj_type == ObjectType.RETURNED and obj.defined_by_module is not None:
base = self._gen_object_file_id(obj.defined_by_module)
return f'{base}.{obj.name}'
return f'root.{_OBJ_ID_MAP[obj.obj_type]}.{obj.name}'
def _link_to_object(self, obj: T.Union[Function, Object], text: T.Optional[str] = None) -> str:
'''
Generate a link to the function/method/object documentation.
The generated link is an HTML link (<a href="">text</a>) instead of
a Markdown link, so that the generated links can be used in custom
(or rather manual) code blocks.
'''
if isinstance(obj, Object):
text = text or f'<ins><code>{obj.name}</code></ins>'
link = self._gen_filename(self._gen_object_file_id(obj), extension="html")
elif isinstance(obj, Method):
text = text or f'<ins><code>`{obj.obj.name}.{obj.name}()`</code></ins>'
file = self._gen_filename(self._gen_object_file_id(obj.obj), extension="html")
link = f'{file}#{obj.obj.name}{obj.name}'
elif isinstance(obj, Function):
text = text or f'<ins><code>`{obj.name}()`</code></ins>'
link = f'{self._gen_filename("root.functions", extension="html")}#{obj.name}'
else:
raise RuntimeError(f'Invalid argument {obj}')
return f'<a href="{link}">{text}</a>'
def _write_file(self, data: str, file_id: str) -> None:#
''' Write the data to disk.
Additionally, links of the form [[function]] are automatically replaced
with valid links to the correct URL. To reference objects / types use the
[[@object]] syntax.
Placeholders with the syntax [[!file_id]] will be replaced with the
corresponding generated markdown file.
'''
self.generated_files[file_id] = self._gen_filename(file_id)
# Replace [[func_name]] and [[obj.method_name]] with links
link_regex = re.compile(r'\[\[[^\]]+\]\]')
matches = link_regex.findall(data)
for i in matches:
obj_id: str = i[2:-2]
if obj_id.startswith('!'):
link_file_id = obj_id[1:]
data = data.replace(i, self._gen_filename(link_file_id))
else:
obj = self._object_from_ref(obj_id)
data = data.replace(i, self._link_to_object(obj))
out_file = self.out_dir / self.generated_files[file_id]
out_file.write_text(data, encoding='ascii')
mlog.log('Generated', mlog.bold(out_file.name))
def _write_template(self, data: T.Dict[str, T.Any], file_id: str, template_name: T.Optional[str] = None) -> None:
''' Render the template mustache files and write the result '''
template_dir = Path(__file__).resolve().parent / 'templates'
template_name = template_name or file_id
template_name = f'{template_name}.mustache'
template_file = template_dir / template_name
# Import here, so that other generators don't also depend on it
import chevron
result = chevron.render(
template=template_file.read_text(encoding='utf-8'),
data=data,
partials_path=template_dir.as_posix(),
warn=True,
)
self._write_file(result, file_id)
# Actual generator functions
def _gen_func_or_method(self, func: Function) -> FunctionDictType:
def render_type(typ: Type) -> str:
def data_type_to_str(dt: DataTypeInfo) -> str:
base = self._link_to_object(dt.data_type, f'<ins>{dt.data_type.name}</ins>')
if dt.holds:
return f'{base}[{render_type(dt.holds)}]'
return base
assert typ.resolved
return ' | '.join([data_type_to_str(x) for x in typ.resolved])
def len_stripped(s: str) -> int:
return len(re.sub(r'<[^>]+>', '', s))
def render_signature() -> str:
# Skip a lot of computations if the function does not take any arguments
if not any([func.posargs, func.optargs, func.kwargs, func.varargs]):
return f'{render_type(func.returns)} {func.name}()'
signature = dedent(f'''\
# {self.brief(func)}
{render_type(func.returns)} {func.name}(
''')
# Calculate maximum lengths of the type and name
all_args: T.List[ArgBase] = []
all_args += func.posargs
all_args += func.optargs
all_args += [func.varargs] if func.varargs else []
max_type_len = 0
max_name_len = 0
if all_args:
max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
max_name_len = max([len(x.name) for x in all_args])
# Generate some common strings
def prepare(arg: ArgBase) -> T.Tuple[str, str, str, str]:
type_str = render_type(arg.type)
type_len = len_stripped(type_str)
type_space = ' ' * (max_type_len - type_len)
name_space = ' ' * (max_name_len - len(arg.name))
name_str = f'<b>{arg.name.replace("<", "&lt;").replace(">", "&gt;")}</b>'
return type_str, type_space, name_str, name_space
for i in func.posargs:
type_str, type_space, name_str, name_space = prepare(i)
signature += f' {type_str}{type_space} {name_str},{name_space} # {self.brief(i)}\n'
for i in func.optargs:
type_str, type_space, name_str, name_space = prepare(i)
signature += f' {type_str}{type_space} [{name_str}],{name_space} # {self.brief(i)}\n'
if func.varargs:
type_str, type_space, name_str, name_space = prepare(func.varargs)
signature += f' {type_str}{type_space} {name_str}...,{name_space} # {self.brief(func.varargs)}\n'
# Abort if there are no kwargs
if not func.kwargs:
return signature + ')'
# Only add this seperator if there are any posargs
if all_args:
signature += '\n # Keyword arguments:\n'
# Recalculate lengths for kwargs
all_args = list(func.kwargs.values())
max_type_len = max([len_stripped(render_type(x.type)) for x in all_args])
max_name_len = max([len(x.name) for x in all_args])
for kwarg in self.sorted_and_filtered(list(func.kwargs.values())):
type_str, type_space, name_str, name_space = prepare(kwarg)
required = ' <i>[required]</i> ' if kwarg.required else ' '
required = required if any([x.required for x in func.kwargs.values()]) else ''
signature += f' {name_str}{name_space} : {type_str}{type_space} {required} # {self.brief(kwarg)}\n'
return signature + ')'
def gen_arg_data(arg: T.Union[PosArg, Kwarg, VarArgs], *, optional: bool = False) -> T.Dict[str, PlaceholderTypes]:
data: T.Dict[str, PlaceholderTypes] = {
'name': arg.name,
'type': render_type(arg.type),
'description': arg.description,
'since': arg.since or None,
'deprecated': arg.deprecated or None,
'optional': optional,
'default': None,
}
if isinstance(arg, VarArgs):
data.update({
'min': str(arg.min_varargs) if arg.min_varargs > 0 else '0',
'max': str(arg.max_varargs) if arg.max_varargs > 0 else 'infinity',
})
if isinstance(arg, (Kwarg, PosArg)):
data.update({'default': arg.default or None})
if isinstance(arg, Kwarg):
data.update({'required': arg.required})
return data
mname = f'\\{func.name}' if func.name == '[index]' else func.name
data: FunctionDictType = {
'name': f'{func.obj.name}.{mname}' if isinstance(func, Method) else func.name,
'base_level': '##' if isinstance(func, Method) else '#',
'type_name_upper': 'Method' if isinstance(func, Method) else 'Function',
'type_name': 'method' if isinstance(func, Method) else 'function',
'description': func.description,
'notes': func.notes,
'warnings': func.warnings,
'example': func.example or None,
'signature_level': 'h4' if isinstance(func, Method) else 'h3',
'signature': render_signature(),
'has_args': bool(func.posargs or func.optargs or func.kwargs or func.varargs),
# Merge posargs and optargs by generating the *[optional]* tag for optargs
'posargs': {
'args': [gen_arg_data(x) for x in func.posargs] + [gen_arg_data(x, optional=True) for x in func.optargs]
} if func.posargs or func.optargs else None,
'kwargs': {'args': [gen_arg_data(x) for x in self.sorted_and_filtered(list(func.kwargs.values()))]} if func.kwargs else None,
'varargs': gen_arg_data(func.varargs) if func.varargs else None,
# For the feature taggs template
'since': func.since or None,
'deprecated': func.deprecated or None,
'optional': False,
'default': None
}
return data
def _write_object(self, obj: Object) -> None:
data = {
'name': obj.name,
'description': obj.description,
'notes': obj.notes,
'warnings': obj.warnings,
'long_name': obj.long_name,
'obj_type_name': _OBJ_ID_MAP[obj.obj_type].capitalize(),
'example': obj.example or None,
'has_methods': bool(obj.methods),
'has_inherited_methods': bool(obj.inherited_methods),
'has_subclasses': bool(obj.extended_by),
'is_returned': bool(obj.returned_by),
'extends': obj.extends_obj.name if obj.extends_obj else None,
'returned_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.returned_by)],
'extended_by': [self._link_to_object(x) for x in self.sorted_and_filtered(obj.extended_by)],
'methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.methods)],
'inherited_methods': [self._gen_func_or_method(m) for m in self.sorted_and_filtered(obj.inherited_methods)],
}
self._write_template(data, self._gen_object_file_id(obj), 'object')
def _write_functions(self) -> None:
data = {'functions': [self._gen_func_or_method(x) for x in self.functions]}
self._write_template(data, 'root.functions')
def _root_refman_docs(self) -> None:
def gen_obj_links(objs: T.List[Object]) -> T.List[T.Dict[str, str]]:
ret: T.List[T.Dict[str, str]] = []
for o in objs:
ret += [{'indent': '', 'link': self._link_to_object(o), 'brief': self.brief(o)}]
for m in self.sorted_and_filtered(o.methods):
ret += [{'indent': ' ', 'link': self._link_to_object(m), 'brief': self.brief(m)}]
if o.obj_type == ObjectType.MODULE and self.extract_returned_by_module(o):
tmp = gen_obj_links(self.extract_returned_by_module(o))
tmp = [{**x, 'indent': ' ' + x['indent']} for x in tmp]
ret += [{'indent': ' ', 'link': '**New objects:**', 'brief': ''}]
ret += [*tmp]
return ret
data = {
'elementary': gen_obj_links(self.elementary),
'returned': gen_obj_links(self.returned),
'builtins': gen_obj_links(self.builtins),
'modules': gen_obj_links(self.modules),
'functions': [{'indent': '', 'link': self._link_to_object(x), 'brief': self.brief(x)} for x in self.functions],
}
self._write_template(data, 'root')
self._write_template({'name': 'Elementary types'}, f'root.{_OBJ_ID_MAP[ObjectType.ELEMENTARY]}', 'dummy')
self._write_template({'name': 'Builtin objects'}, f'root.{_OBJ_ID_MAP[ObjectType.BUILTIN]}', 'dummy')
self._write_template({'name': 'Returned objects'}, f'root.{_OBJ_ID_MAP[ObjectType.RETURNED]}', 'dummy')
self._write_template({'name': 'Modules'}, f'root.{_OBJ_ID_MAP[ObjectType.MODULE]}', 'dummy')
def generate(self) -> None:
mlog.log('Generating markdown files...')
with mlog.nested():
self._write_functions()
for obj in self.objects:
self._write_object(obj)
self._root_refman_docs()
self._configure_sitemap()
def _configure_sitemap(self) -> None:
'''
Replaces the `@REFMAN_PLACEHOLDER@` placeholder with the reference
manual sitemap. The structure of the sitemap is derived from the
file IDs.
'''
raw = self.sitemap_in.read_text(encoding='utf-8')
out = ''
for l in raw.split('\n'):
if '@REFMAN_PLACEHOLDER@' not in l:
out += f'{l}\n'
continue
mlog.log('Generating', mlog.bold(self.sitemap_out.as_posix()))
base_indent = l.replace('@REFMAN_PLACEHOLDER@', '')
for k in sorted(self.generated_files.keys()):
indent = base_indent + '\t' * k.count('.')
out += f'{indent}{self.generated_files[k]}\n'
self.sitemap_out.write_text(out, encoding='utf-8')

@ -24,6 +24,7 @@ from .loaderyaml import LoaderYAML
from .generatorbase import GeneratorBase
from .generatorprint import GeneratorPrint
from .generatorpickle import GeneratorPickle
from .generatormd import GeneratorMD
meson_root = Path(__file__).absolute().parents[2]
@ -31,6 +32,7 @@ def main() -> int:
parser = argparse.ArgumentParser(description='Meson reference manual generator')
parser.add_argument('-l', '--loader', type=str, default='yaml', choices=['yaml'], help='Information loader backend')
parser.add_argument('-g', '--generator', type=str, choices=['print', 'pickle', 'md'], required=True, help='Generator backend')
parser.add_argument('-s', '--sitemap', type=Path, default=meson_root / 'docs' / 'sitemap.txt', help='Path to the input sitemap.txt')
parser.add_argument('-o', '--out', type=Path, required=True, help='Output directory for generated files')
parser.add_argument('--depfile', type=Path, default=None, help='Set to generate a depfile')
parser.add_argument('--force-color', action='store_true', help='Force enable colors')
@ -49,6 +51,7 @@ def main() -> int:
generators: T.Dict[str, T.Callable[[], GeneratorBase]] = {
'print': lambda: GeneratorPrint(refMan),
'pickle': lambda: GeneratorPickle(refMan, args.out),
'md': lambda: GeneratorMD(refMan, args.out, args.sitemap),
}
generator = generators[args.generator]()

@ -0,0 +1,28 @@
<!-- Hotdoc / markdown inserts <p> around the table elements. Override the margin to make them invisible -->
<style>
.nomargin p:last-child { margin-bottom: 0px !important; }
</style>
<table>
<thead>
<tr>
<th style="white-space: nowrap; text-align: center;">Name</th>
<th style="white-space: nowrap; text-align: center;">Type</th>
<th style="width: 56%;">Description</th>
<th style="white-space: nowrap; text-align: center; width: 0px;">Tags</th>
</tr>
</thead>
<tbody class="nomargin">
{{#args}}
<tr>
<td style="white-space: nowrap; text-align: center; padding: 6px;"><code class="language-meson">{{name}}</code></td>
<td style="white-space: revert; text-align: center; padding: 6px; word-wrap: break-word;">{{&type}}</td>
<!-- This suboptimal formating is required to ensure hotdoc correctly generates the HTML -->
<td style="width: 56%; padding: 6px;">
{{&description}}
</td>
<td style="white-space: nowrap; text-align: center; padding: 6px; width: 0px;">{{>taggs}}</td>
</tr>
{{/args}}
</tbody>
</table>

@ -0,0 +1,8 @@
---
short-description: {{name}}
render-subpages: true
...
# {{name}}
See the [root manual document]([[!root]]) for
a general overview.

@ -0,0 +1,55 @@
{{base_level}}# {{name}}()
{{&description}}
<p style="padding: 5px; margin: 0px;"></p> <!-- A bit of space because we remove the top margin below -->
<div style="display: flex;">
<{{signature_level}} style="margin-top: 0px;">Signature</{{signature_level}}>
<div style="flex-grow: 1;"></div>
<div>{{>taggs}}</div>
</div>
<pre><code class="language-meson">{{&signature}}</code></pre>
{{#example}}
<p style="padding: 5px; margin: 0px;"></p> <!-- A bit more space -->
{{base_level}}## Example
{{&example}}
{{/example}}
{{>notes}}
{{#has_args}}
<p style="padding: 5px; margin: 0px;"></p> <!-- A bit more space -->
{{base_level}}## Arguments
{{/has_args}}
{{#posargs}}
The {{type_name}} `{{name}}()` accepts the following positional arguments:
{{>args}}
<p style="padding: 5px; margin: 0px;"></p> <!-- Extra space -->
{{/posargs}}
{{#varargs}}
{{#posargs}}Additionally, the{{/posargs}}{{^posargs}}The{{/posargs}}
{{type_name}} accepts between `{{min}}` and `{{max}}` variadic
arguments (`{{name}}...`) of type <code>{{&type}}</code>.
{{&description}}
{{>taggs}}
<p style="padding: 5px; margin: 0px;"></p> <!-- Extra space -->
{{/varargs}}
{{#kwargs}}
{{#posargs}}Finally, `{{name}}()`{{/posargs}}{{^posargs}}The {{type_name}} `{{name}}()`{{/posargs}}
accepts the following keyword arguments:
{{>args}}
{{/kwargs}}

@ -0,0 +1,14 @@
{{#notes}}
<div class="alert alert-info">
<strong>Note:</strong>
{{&.}}
</div>
{{/notes}}
{{#warnings}}
<div class="alert alert-warning">
<strong>Warning:</strong>
{{&.}}
</div>
{{/warnings}}

@ -0,0 +1,62 @@
---
short-description: "{{obj_type_name}} object: {{long_name}}"
title: {{name}}{{#extends}} (extends {{.}}){{/extends}}
render-subpages: false
...
# {{long_name}} (`{{name}}`{{#extends}} extends [[@{{.}}]]{{/extends}})
{{&description}}
{{#has_subclasses}}
## Extended by
{{long_name}} is extended by the following subtypes:
{{#extended_by}}
- {{&.}}
{{/extended_by}}
{{/has_subclasses}}
{{#is_returned}}
## Returned by
{{long_name}} objects are returned by the following functions and methods:
{{#returned_by}}
- {{&.}}
{{/returned_by}}
{{/is_returned}}
{{#example}}
## Example
<pre><code class="language-meson">{{&example}}</code></pre>
{{/example}}
{{>notes}}
{{#has_methods}}
## {{long_name}} methods
{{#methods}}
<p style="padding: 7.5px; margin: 0px;"></p>
{{>func}}
<p style="padding: 7.5px; margin: 0px;"></p>
---
{{/methods}}
{{/has_methods}}
{{#has_inherited_methods}}
## Inherited methods
{{#inherited_methods}}
<p style="padding: 7.5px; margin: 0px;"></p>
{{>func}}
<p style="padding: 7.5px; margin: 0px;"></p>
---
{{/inherited_methods}}
{{/has_inherited_methods}}

@ -0,0 +1,20 @@
---
short-description: Meson functions
render-subpages: false
...
# Functions
This document lists all functions available in `meson.build` files.
See the [root manual document]([[!root]]) for
an overview of all features.
{{#functions}}
<p style="padding: 7.5px; margin: 0px;"></p>
{{>func}}
<p style="padding: 7.5px; margin: 0px;"></p>
---
{{/functions}}

@ -0,0 +1,48 @@
---
short-description: The Meson reference manual
render-subpages: false
...
# Reference manual
This is the root page of the Meson reference manual. All functions
and methods are documented in detail in the following subpages:
## Elementary types
{{#elementary}}
{{>root_link}}
{{/elementary}}
## Functions
The following functions are available in build files. Click on each
to see the description and usage. The objects returned by them are
[listed here](#returned-objects).
{{#functions}}
{{>root_link}}
{{/functions}}
## Builtin objects
These are built-in objects that are always available.
{{#builtins}}
{{>root_link}}
{{/builtins}}
## Returned objects
These are objects that can be returned by [functions](#functions)
or other methods.
{{#returned}}
{{>root_link}}
{{/returned}}
## Modules
{{#modules}}
{{>root_link}}
{{/modules}}

@ -0,0 +1 @@
{{indent}}- <span style="display: flex;"><span>{{&link}}</span><span style="flex-grow: 1;"></span><span style="text-align: right;">{{&brief}}</span></span>

@ -0,0 +1,18 @@
<div style="margin: 0px; text-align: center;">
{{#since}}
<p style="margin: 0px;"><em style="color: #5affff;">(since {{since}})</em></p>
{{/since}}
{{#deprecated}}
<div style="color: #ffa844">
<p style="margin: 0px;"><strong>DEPRECATED</strong></p>
<p style="margin: 0px;"><em>in {{deprecated}}</em></p>
</div>
{{/deprecated}}
{{#optional}}
<p style="margin: 0px;"><b>[optional]</b></p>
{{/optional}}
{{#default}}
<p style="margin: 0px;"><code class="language-meson">default =
{{default}}</code></p>
{{/default}}
</div>
Loading…
Cancel
Save