Merge pull request #9377 from mensinda/jsonDocs
docs: Add a JSON documentation backendpull/9445/head
Jussi Pakkanen
3 years ago
committed by
GitHub
commit
475b8b1ad8
10 changed files with 511 additions and 17 deletions
@ -0,0 +1,195 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
# SPDX-License-Identifer: Apache-2.0 |
||||
# Copyright 2021 The Meson development team |
||||
|
||||
import argparse |
||||
import json |
||||
from pathlib import Path |
||||
from copy import deepcopy |
||||
|
||||
import typing as T |
||||
|
||||
T_None = type(None) |
||||
|
||||
# Global root object |
||||
root: dict |
||||
|
||||
def assert_has_typed_keys(path: str, data: dict, keys: T.Dict[str, T.Any]) -> dict: |
||||
assert set(data.keys()).issuperset(keys.keys()), f'{path}: DIFF: {set(data.keys()).difference(keys.keys())}' |
||||
res = dict() |
||||
for key, val in keys.items(): |
||||
cur = data.pop(key) |
||||
assert isinstance(cur, val), f'{path}: type({key}: {cur}) != {val}' |
||||
res[key] = cur |
||||
return res |
||||
|
||||
def validate_base_obj(path: str, name: str, obj: dict) -> None: |
||||
expected: T.Dict[str, T.Any] = { |
||||
'name': str, |
||||
'description': str, |
||||
'since': (str, T_None), |
||||
'deprecated': (str, T_None), |
||||
'notes': list, |
||||
'warnings': list, |
||||
} |
||||
cur = assert_has_typed_keys(f'{path}.{name}', obj, expected) |
||||
assert cur['name'], f'{path}.{name}' |
||||
assert cur['description'], f'{path}.{name}' |
||||
assert cur['name'] == name, f'{path}.{name}' |
||||
assert all(isinstance(x, str) and x for x in cur['notes']), f'{path}.{name}' |
||||
assert all(isinstance(x, str) and x for x in cur['warnings']), f'{path}.{name}' |
||||
|
||||
def validate_type(path: str, typ: dict) -> None: |
||||
expected: T.Dict[str, T.Any] = { |
||||
'obj': str, |
||||
'holds': list, |
||||
} |
||||
cur = assert_has_typed_keys(path, typ, expected) |
||||
assert not typ, f'{path} has extra keys: {typ.keys()}' |
||||
assert cur['obj'] in root['objects'], path |
||||
for i in cur['holds']: |
||||
validate_type(path, i) |
||||
|
||||
def validate_arg(path: str, name: str, arg: dict) -> None: |
||||
validate_base_obj(path, name, arg) |
||||
expected: T.Dict[str, T.Any] = { |
||||
'type': list, |
||||
'type_str': str, |
||||
'required': bool, |
||||
'default': (str, T_None), |
||||
'min_varargs': (int, T_None), |
||||
'max_varargs': (int, T_None), |
||||
} |
||||
cur = assert_has_typed_keys(f'{path}.{name}', arg, expected) |
||||
assert not arg, f'{path}.{name} has extra keys: {arg.keys()}' |
||||
assert cur['type'], f'{path}.{name}' |
||||
assert cur['type_str'], f'{path}.{name}' |
||||
for i in cur['type']: |
||||
validate_type(f'{path}.{name}', i) |
||||
if cur['min_varargs'] is not None: |
||||
assert cur['min_varargs'] > 0, f'{path}.{name}' |
||||
if cur['max_varargs'] is not None: |
||||
assert cur['max_varargs'] > 0, f'{path}.{name}' |
||||
|
||||
def validate_function(path: str, name: str, func: dict) -> None: |
||||
validate_base_obj(path, name, func) |
||||
expected: T.Dict[str, T.Any] = { |
||||
'returns': list, |
||||
'returns_str': str, |
||||
'example': (str, T_None), |
||||
'posargs': dict, |
||||
'optargs': dict, |
||||
'kwargs': dict, |
||||
'varargs': (dict, T_None), |
||||
} |
||||
cur = assert_has_typed_keys(f'{path}.{name}', func, expected) |
||||
assert not func, f'{path}.{name} has extra keys: {func.keys()}' |
||||
assert cur['returns'], f'{path}.{name}' |
||||
assert cur['returns_str'], f'{path}.{name}' |
||||
for i in cur['returns']: |
||||
validate_type(f'{path}.{name}', i) |
||||
for k, v in cur['posargs'].items(): |
||||
validate_arg(f'{path}.{name}', k, v) |
||||
for k, v in cur['optargs'].items(): |
||||
validate_arg(f'{path}.{name}', k, v) |
||||
for k, v in cur['kwargs'].items(): |
||||
validate_arg(f'{path}.{name}', k, v) |
||||
if cur['varargs']: |
||||
validate_arg(f'{path}.{name}', cur['varargs']['name'], cur['varargs']) |
||||
|
||||
def validate_object(path: str, name: str, obj: dict) -> None: |
||||
validate_base_obj(path, name, obj) |
||||
expected: T.Dict[str, T.Any] = { |
||||
'example': (str, T_None), |
||||
'object_type': str, |
||||
'methods': dict, |
||||
'is_container': bool, |
||||
'extends': (str, T_None), |
||||
'returned_by': list, |
||||
'extended_by': list, |
||||
'defined_by_module': (str, T_None), |
||||
} |
||||
cur = assert_has_typed_keys(f'{path}.{name}', obj, expected) |
||||
assert not obj, f'{path}.{name} has extra keys: {obj.keys()}' |
||||
for key, val in cur['methods'].items(): |
||||
validate_function(f'{path}.{name}', key, val) |
||||
if cur['extends'] is not None: |
||||
assert cur['extends'] in root['objects'], f'{path}.{name}' |
||||
assert all(isinstance(x, str) for x in cur['returned_by']), f'{path}.{name}' |
||||
assert all(isinstance(x, str) for x in cur['extended_by']), f'{path}.{name}' |
||||
assert all(x in root['objects'] for x in cur['extended_by']), f'{path}.{name}' |
||||
if cur['defined_by_module'] is not None: |
||||
assert cur['defined_by_module'] in root['objects'], f'{path}.{name}' |
||||
assert cur['object_type'] == 'RETURNED', f'{path}.{name}' |
||||
assert root['objects'][cur['defined_by_module']]['object_type'] == 'MODULE', f'{path}.{name}' |
||||
assert name in root['objects_by_type']['modules'][cur['defined_by_module']], f'{path}.{name}' |
||||
return |
||||
assert cur['object_type'] in {'ELEMENTARY', 'BUILTIN', 'MODULE', 'RETURNED'}, f'{path}.{name}' |
||||
if cur['object_type'] == 'ELEMENTARY': |
||||
assert name in root['objects_by_type']['elementary'], f'{path}.{name}' |
||||
if cur['object_type'] == 'BUILTIN': |
||||
assert name in root['objects_by_type']['builtins'], f'{path}.{name}' |
||||
if cur['object_type'] == 'RETURNED': |
||||
assert name in root['objects_by_type']['returned'], f'{path}.{name}' |
||||
if cur['object_type'] == 'MODULE': |
||||
assert name in root['objects_by_type']['modules'], f'{path}.{name}' |
||||
|
||||
def main() -> int: |
||||
global root |
||||
|
||||
parser = argparse.ArgumentParser(description='Meson JSON docs validator') |
||||
parser.add_argument('doc_file', type=Path, help='The JSON docs to validate') |
||||
args = parser.parse_args() |
||||
|
||||
root_tmp = json.loads(args.doc_file.read_text(encoding='utf-8')) |
||||
root = deepcopy(root_tmp) |
||||
assert isinstance(root, dict) |
||||
|
||||
expected: T.Dict[str, T.Any] = { |
||||
'version_major': int, |
||||
'version_minor': int, |
||||
'meson_version': str, |
||||
'functions': dict, |
||||
'objects': dict, |
||||
'objects_by_type': dict, |
||||
} |
||||
cur = assert_has_typed_keys('root', root_tmp, expected) |
||||
assert not root_tmp, f'root has extra keys: {root_tmp.keys()}' |
||||
|
||||
refs = cur['objects_by_type'] |
||||
expected = { |
||||
'elementary': list, |
||||
'builtins': list, |
||||
'returned': list, |
||||
'modules': dict, |
||||
} |
||||
assert_has_typed_keys(f'root.objects_by_type', refs, expected) |
||||
assert not refs, f'root.objects_by_type has extra keys: {refs.keys()}' |
||||
assert all(isinstance(x, str) for x in root['objects_by_type']['elementary']) |
||||
assert all(isinstance(x, str) for x in root['objects_by_type']['builtins']) |
||||
assert all(isinstance(x, str) for x in root['objects_by_type']['returned']) |
||||
assert all(isinstance(x, str) for x in root['objects_by_type']['modules']) |
||||
assert all(x in root['objects'] for x in root['objects_by_type']['elementary']) |
||||
assert all(x in root['objects'] for x in root['objects_by_type']['builtins']) |
||||
assert all(x in root['objects'] for x in root['objects_by_type']['returned']) |
||||
assert all(x in root['objects'] for x in root['objects_by_type']['modules']) |
||||
assert all(root['objects'][x]['object_type'] == 'ELEMENTARY' for x in root['objects_by_type']['elementary']) |
||||
assert all(root['objects'][x]['object_type'] == 'BUILTIN' for x in root['objects_by_type']['builtins']) |
||||
assert all(root['objects'][x]['object_type'] == 'RETURNED' for x in root['objects_by_type']['returned']) |
||||
assert all(root['objects'][x]['object_type'] == 'MODULE' for x in root['objects_by_type']['modules']) |
||||
|
||||
# Check that module references are correct |
||||
assert all(all(isinstance(x, str) for x in v) for k, v in root['objects_by_type']['modules'].items()) |
||||
assert all(all(x in root['objects'] for x in v) for k, v in root['objects_by_type']['modules'].items()) |
||||
assert all(all(root['objects'][x]['defined_by_module'] == k for x in v) for k, v in root['objects_by_type']['modules'].items()) |
||||
|
||||
for key, val in cur['functions'].items(): |
||||
validate_function('root', key, val) |
||||
for key, val in cur['objects'].items(): |
||||
validate_object('root', key, val) |
||||
|
||||
return 0 |
||||
|
||||
if __name__ == '__main__': |
||||
raise SystemExit(main()) |
||||
@ -0,0 +1,120 @@ |
||||
# SPDX-License-Identifer: Apache-2.0 |
||||
# Copyright 2021 The Meson development team |
||||
|
||||
from pathlib import Path |
||||
import json |
||||
import re |
||||
|
||||
from .generatorbase import GeneratorBase |
||||
from . import jsonschema as J |
||||
from .model import ( |
||||
ReferenceManual, |
||||
Function, |
||||
Object, |
||||
Type, |
||||
|
||||
PosArg, |
||||
VarArgs, |
||||
Kwarg, |
||||
) |
||||
|
||||
import typing as T |
||||
|
||||
class GeneratorJSON(GeneratorBase): |
||||
def __init__(self, manual: ReferenceManual, out: Path, enable_modules: bool) -> None: |
||||
super().__init__(manual) |
||||
self.out = out |
||||
self.enable_modules = enable_modules |
||||
|
||||
def _generate_type(self, typ: Type) -> T.List[J.Type]: |
||||
return [ |
||||
{ |
||||
'obj': x.data_type.name, |
||||
'holds': self._generate_type(x.holds) if x.holds else [], |
||||
} |
||||
for x in typ.resolved |
||||
] |
||||
|
||||
def _generate_type_str(self, typ: Type) -> str: |
||||
# Remove all whitespaces |
||||
return re.sub(r'[ \n\r\t]', '', typ.raw) |
||||
|
||||
def _generate_arg(self, arg: T.Union[PosArg, VarArgs, Kwarg], isOptarg: bool = False) -> J.Argument: |
||||
return { |
||||
'name': arg.name, |
||||
'description': arg.description, |
||||
'since': arg.since if arg.since else None, |
||||
'deprecated': arg.deprecated if arg.deprecated else None, |
||||
'type': self._generate_type(arg.type), |
||||
'type_str': self._generate_type_str(arg.type), |
||||
'required': arg.required if isinstance(arg, Kwarg) else not isOptarg and not isinstance(arg, VarArgs), |
||||
'default': arg.default if isinstance(arg, (PosArg, Kwarg)) else None, |
||||
'min_varargs': arg.min_varargs if isinstance(arg, VarArgs) and arg.min_varargs > 0 else None, |
||||
'max_varargs': arg.max_varargs if isinstance(arg, VarArgs) and arg.max_varargs > 0 else None, |
||||
|
||||
# Not yet supported |
||||
'notes': [], |
||||
'warnings': [], |
||||
} |
||||
|
||||
def _generate_function(self, func: Function) -> J.Function: |
||||
return { |
||||
'name': func.name, |
||||
'description': func.description, |
||||
'since': func.since if func.since else None, |
||||
'deprecated': func.deprecated if func.deprecated else None, |
||||
'notes': func.notes, |
||||
'warnings': func.warnings, |
||||
'example': func.example if func.example else None, |
||||
'returns': self._generate_type(func.returns), |
||||
'returns_str': self._generate_type_str(func.returns), |
||||
'posargs': {x.name: self._generate_arg(x) for x in func.posargs}, |
||||
'optargs': {x.name: self._generate_arg(x, True) for x in func.optargs}, |
||||
'kwargs': {x.name: self._generate_arg(x) for x in self.sorted_and_filtered(list(func.kwargs.values()))}, |
||||
'varargs': self._generate_arg(func.varargs) if func.varargs else None, |
||||
} |
||||
|
||||
def _generate_objects(self, obj: Object) -> J.Object: |
||||
return { |
||||
'name': obj.name, |
||||
'description': obj.description, |
||||
'since': obj.since if obj.since else None, |
||||
'deprecated': obj.deprecated if obj.deprecated else None, |
||||
'notes': obj.notes, |
||||
'warnings': obj.warnings, |
||||
'defined_by_module': obj.defined_by_module.name if obj.defined_by_module else None, |
||||
'object_type': obj.obj_type.name, |
||||
'is_container': obj.is_container, |
||||
'example': obj.example if obj.example else None, |
||||
'extends': obj.extends if obj.extends else None, |
||||
'returned_by': [x.name for x in self.sorted_and_filtered(obj.returned_by)], |
||||
'extended_by': [x.name for x in self.sorted_and_filtered(obj.extended_by)], |
||||
'methods': {x.name: self._generate_function(x) for x in self.sorted_and_filtered(obj.methods)}, |
||||
} |
||||
|
||||
def _extract_meson_version(self) -> str: |
||||
# Hack around python relative imports to get to the Meson version |
||||
import sys |
||||
sys.path.append(Path(__file__).resolve().parents[2].as_posix()) |
||||
from mesonbuild.coredata import version |
||||
return version |
||||
|
||||
def generate(self) -> None: |
||||
data: J.Root = { |
||||
'version_major': J.VERSION_MAJOR, |
||||
'version_minor': J.VERSION_MINOR, |
||||
'meson_version': self._extract_meson_version(), |
||||
'functions': {x.name: self._generate_function(x) for x in self.sorted_and_filtered(self.functions)}, |
||||
'objects': {x.name: self._generate_objects(x) for x in self.sorted_and_filtered(self.objects)}, |
||||
'objects_by_type': { |
||||
'elementary': [x.name for x in self.elementary], |
||||
'builtins': [x.name for x in self.builtins], |
||||
'returned': [x.name for x in self.returned], |
||||
'modules': { |
||||
x.name: [y.name for y in self.sorted_and_filtered(self.extract_returned_by_module(x))] |
||||
for x in self.modules |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
self.out.write_text(json.dumps(data), encoding='utf-8') |
||||
@ -0,0 +1,87 @@ |
||||
# SPDX-License-Identifer: Apache-2.0 |
||||
# Copyright 2021 The Meson development team |
||||
|
||||
import typing as T |
||||
|
||||
# The following variables define the current version of |
||||
# the JSON documentation format. This is different from |
||||
# the Meson version |
||||
|
||||
VERSION_MAJOR = 1 # Changes here indicate breaking format changes (changes to existing keys) |
||||
VERSION_MINOR = 0 # Changes here indicate non-breaking changes (only new keys are added to the existing structure) |
||||
|
||||
class BaseObject(T.TypedDict): |
||||
''' |
||||
Base object for most dicts in the JSON doc. |
||||
|
||||
All objects inheriting from BaseObject will support |
||||
the keys specified here: |
||||
''' |
||||
name: str |
||||
description: str |
||||
since: T.Optional[str] |
||||
deprecated: T.Optional[str] |
||||
notes: T.List[str] |
||||
warnings: T.List[str] |
||||
|
||||
class Type(T.TypedDict): |
||||
obj: str # References an object from `root.objects` |
||||
holds: T.Sequence[object] # Mypy does not support recusive dicts, but this should be T.List[Type]... |
||||
|
||||
class Argument(BaseObject): |
||||
''' |
||||
Object that represents any type of a single function or method argumet. |
||||
''' |
||||
type: T.List[Type] # A non-empty list of types that are supported. |
||||
type_str: str # Formated version of `type`. Is guranteed to not contain any whitespaces. |
||||
required: bool |
||||
default: T.Optional[str] |
||||
min_varargs: T.Optional[int] # Only relevant for varargs, must be `null` for all other types of arguments |
||||
max_varargs: T.Optional[int] # Only relevant for varargs, must be `null` for all other types of arguments |
||||
|
||||
class Function(BaseObject): |
||||
''' |
||||
Represents a function or method. |
||||
''' |
||||
returns: T.List[Type] # A non-empty list of types that are supported. |
||||
returns_str: str # Formated version of `returns`. Is guranteed to not contain any whitespaces. |
||||
example: T.Optional[str] |
||||
posargs: T.Dict[str, Argument] |
||||
optargs: T.Dict[str, Argument] |
||||
kwargs: T.Dict[str, Argument] |
||||
varargs: T.Optional[Argument] |
||||
|
||||
class Object(BaseObject): |
||||
''' |
||||
Represents all types of Meson objects. The specific object type is stored in the `object_type` field. |
||||
''' |
||||
example: T.Optional[str] |
||||
object_type: str # Defines the object type: Must be one of: ELEMENTARY, BUILTIN, MODULE, RETURNED |
||||
methods: T.Dict[str, Function] |
||||
is_container: bool |
||||
extends: T.Optional[str] |
||||
returned_by: T.List[str] |
||||
extended_by: T.List[str] |
||||
defined_by_module: T.Optional[str] |
||||
|
||||
class ObjectsByType(T.TypedDict): |
||||
''' |
||||
References to other objects are stored here for ease of navigation / filtering |
||||
''' |
||||
elementary: T.List[str] |
||||
builtins: T.List[str] |
||||
returned: T.List[str] |
||||
modules: T.Dict[str, T.List[str]] |
||||
|
||||
|
||||
|
||||
class Root(T.TypedDict): |
||||
''' |
||||
The root object of the JSON reference manual |
||||
''' |
||||
version_major: int # See the description above for |
||||
version_minor: int # VERSION_MAJOR and VERSION_MINOR |
||||
meson_version: str |
||||
functions: T.Dict[str, Function] # A mapping of <name> to a `Function` object for *all* Meson functions |
||||
objects: T.Dict[str, Object] # A mapping of <name> to a `Object` object for *all* Meson objects (including modules, elementary, etc.) |
||||
objects_by_type: ObjectsByType |
||||
@ -0,0 +1,22 @@ |
||||
# SPDX-License-Identifer: Apache-2.0 |
||||
# Copyright 2021 The Meson development team |
||||
|
||||
from pathlib import Path |
||||
import pickle |
||||
|
||||
from .loaderbase import LoaderBase |
||||
from .model import ReferenceManual |
||||
|
||||
class LoaderPickle(LoaderBase): |
||||
def __init__(self, in_file: Path) -> None: |
||||
super().__init__() |
||||
self.in_file = in_file |
||||
|
||||
def load_impl(self) -> ReferenceManual: |
||||
res = pickle.loads(self.in_file.read_bytes()) |
||||
assert isinstance(res, ReferenceManual) |
||||
return res |
||||
|
||||
# Assume that the pickled data is OK and skip validation |
||||
def load(self) -> ReferenceManual: |
||||
return self.load_impl() |
||||
Loading…
Reference in new issue