protodoc: additional features to generate base.proto. (#215)

* Support for enums, nested messages, oneof and map types. * External links for scalar and WKT types. Signed-off-by: Harvey Tuch <htuch@google.com>
8 years ago · 978688d58b
parent bacf23f51f
commit 978688d58b
3 changed files with 203 additions and 60 deletions
--- a/api/base.proto
+++ b/api/base.proto
@ -54,7 +54,8 @@ message Endpoint {
 // Endpoints have a Metadata object associated and routes contain a Metadata
 // object to match against. There are some well defined metadata used today for
 // this purpose:
-// - {"envoy.lb": {"canary": <bool> }}. This indicates the canary status of an
+//
+// * {"envoy.lb": {"canary": <bool> }}. This indicates the canary status of an
 //   endpoint and is also used during header processing
 //   (x-envoy-upstream-canary) and for stats purposes.
 message Metadata {
--- a/docs/build.sh
+++ b/docs/build.sh
@ -28,6 +28,7 @@ bazel --batch build -s ${BAZEL_BUILD_OPTIONS} //api --aspects \
 # TODO(htuch): Factor this out of this script.
 PROTO_RST="
  /api/address/api/address.proto.rst
+  /api/base/api/base.proto.rst
 "

 # Only copy in the protos we care about and know how to deal with in protodoc.
--- a/tools/protodoc/protodoc.py
+++ b/tools/protodoc/protodoc.py
@ -1,7 +1,10 @@
 # protoc plugin to map from FileDescriptorProtos to Envoy doc style RST.
 # See https://github.com/google/protobuf/blob/master/src/google/protobuf/descriptor.proto
-# for the underlying protos mentioned in this file.
+# for the underlying protos mentioned in this file. See
+# http://www.sphinx-doc.org/en/stable/rest.html for Sphinx RST syntax.

+from collections import defaultdict
+import copy
 import functools
 import sys

@ -10,6 +13,12 @@ from google.protobuf.compiler import plugin_pb2
 # Namespace prefix for Envoy APIs.
 ENVOY_API_NAMESPACE_PREFIX = '.envoy.api.v2.'

+# Namespace prefix for WKTs.
+WKT_NAMESPACE_PREFIX = '.google.protobuf.'
+
+# http://www.fileformat.info/info/unicode/char/2063/index.htm
+UNICODE_INVISIBLE_SEPARATOR = u'\u2063'
+

 class ProtodocError(Exception):
  """Base error class for the protodoc module."""
@ -49,6 +58,40 @@ class SourceCodeInfo(object):
    return ''


+class TypeContext(object):
+  """Contextual information for a message/field.
+
+  Provides information around namespaces and enclosing types for fields and
+  nested messages/enums.
+  """
+
+  def __init__(self, source_code_info, path, name):
+    # SourceCodeInfo as per
+    # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto.
+    self.source_code_info = source_code_info
+    # path: a list of path indexes as per
+    #  https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717.
+    #  Extended as nested objects are traversed.
+    self.path = path
+    # Message/enum/field name. Extended as nested objects are traversed.
+    self.name = name
+    # Map from type name to the correct type annotation string, e.g. from
+    # ".envoy.api.v2.Foo.Bar" to "map<string, string>". This is lost during
+    # proto synthesis and is dynamically recovered in FormatMessage.
+    self.map_typenames = {}
+    # Map from a message's oneof index to the fields sharing a oneof.
+    self.oneof_fields = {}
+
+  def Extend(self, path, name):
+    extended = copy.deepcopy(self)
+    extended.path.extend(path)
+    extended.name = '%s.%s' % (self.name, name)
+    return extended
+
+  def LeadingCommentPathLookup(self):
+    return self.source_code_info.LeadingCommentPathLookup(self.path)
+
+
 def MapLines(f, s):
  """Apply a function across each line in a flat string.

@ -71,6 +114,14 @@ def IndentLines(spaces, lines):
  return map(functools.partial(Indent, spaces), lines)


+def FormatInternalLink(text, ref):
+  return ':ref:`%s <%s>`' % (text, ref)
+
+
+def FormatExternalLink(text, ref):
+  return '`%s <%s>`_' % (text, ref)
+
+
 def FormatHeader(style, text):
  """Format RST header.

@ -83,14 +134,17 @@ def FormatHeader(style, text):
  return '%s\n%s\n\n' % (text, style * len(text))


-def FormatFieldTypeAsJson(field):
+def FormatFieldTypeAsJson(type_context, field):
  """Format FieldDescriptorProto.Type as a pseudo-JSON string.

  Args:
+    type_context: contextual information for message/enum/field.
    field: FieldDescriptor proto.
  Return:
    RST formatted pseudo-JSON string representation of field type.
  """
+  if NormalizeFQN(field.type_name) in type_context.map_typenames:
+    return '"{...}"'
  if field.label == field.LABEL_REPEATED:
    return '[]'
  if field.type == field.TYPE_MESSAGE:
@ -98,15 +152,19 @@ def FormatFieldTypeAsJson(field):
  return '"..."'


-def FormatMessageAsJson(msg):
+def FormatMessageAsJson(type_context, msg):
  """Format a message definition DescriptorProto as a pseudo-JSON block.

  Args:
+    type_context: contextual information for message/enum/field.
    msg: message definition DescriptorProto.
  Return:
    RST formatted pseudo-JSON string representation of message definition.
  """
-  lines = ['"%s": %s' % (f.name, FormatFieldTypeAsJson(f)) for f in msg.field]
+  lines = [
+      '"%s": %s' % (f.name, FormatFieldTypeAsJson(type_context, f))
+      for f in msg.field
+  ]
  return '.. code-block:: json\n\n  {\n' + ',\n'.join(IndentLines(
      4, lines)) + '\n  }\n\n'

@ -123,18 +181,6 @@ def NormalizeFQN(fqn):
  """
  if fqn.startswith(ENVOY_API_NAMESPACE_PREFIX):
    return fqn[len(ENVOY_API_NAMESPACE_PREFIX):]
-
-  def Wrapped(s):
-    return '{%s}' % s
-
-  remap_fqn = {
-      '.google.protobuf.UInt32Value': Wrapped('uint32'),
-      '.google.protobuf.UInt64Value': Wrapped('uint64'),
-      '.google.protobuf.BoolValue': Wrapped('bool'),
-  }
-  if fqn in remap_fqn:
-    return remap_fqn[fqn]
-
  return fqn


@ -143,24 +189,35 @@ def FormatEmph(s):
  return '*%s*' % s


-def FormatFieldType(field):
+def FormatFieldType(type_context, field):
  """Format a FieldDescriptorProto type description.

  Adds cross-refs for message types.
  TODO(htuch): Add cross-refs for enums as well.

  Args:
+    type_context: contextual information for message/enum/field.
    field: FieldDescriptor proto.
  Return:
    RST formatted field type.
  """
-  if field.type == field.TYPE_MESSAGE and field.type_name.startswith(
-      ENVOY_API_NAMESPACE_PREFIX):
+  if field.type_name.startswith(ENVOY_API_NAMESPACE_PREFIX):
    type_name = NormalizeFQN(field.type_name)
-    return ':ref:`%s <%s>`' % (type_name, MessageCrossRefLabel(type_name))
-  # TODO(htuch): Replace with enum handling.
-  if field.type_name:
-    return FormatEmph(NormalizeFQN(field.type_name))
+    if field.type == field.TYPE_MESSAGE:
+      if type_context.map_typenames and type_name in type_context.map_typenames:
+        return type_context.map_typenames[type_name]
+      return FormatInternalLink(type_name, MessageCrossRefLabel(type_name))
+    if field.type == field.TYPE_ENUM:
+      return FormatInternalLink(type_name, EnumCrossRefLabel(type_name))
+  elif field.type_name.startswith(WKT_NAMESPACE_PREFIX):
+    wkt = field.type_name[len(WKT_NAMESPACE_PREFIX):]
+    return FormatExternalLink(
+        wkt,
+        'https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#%s'
+        % wkt.lower())
+  elif field.type_name:
+    return field.type_name
+
  pretty_type_names = {
      field.TYPE_DOUBLE: 'double',
      field.TYPE_FLOAT: 'float',
@ -173,7 +230,9 @@ def FormatFieldType(field):
      field.TYPE_BYTES: 'bytes',
  }
  if field.type in pretty_type_names:
-    return FormatEmph(pretty_type_names[field.type])
+    return FormatExternalLink(
+        pretty_type_names[field.type],
+        'https://developers.google.com/protocol-buffers/docs/proto#scalar')
  raise ProtodocError('Unknown field type ' + str(field.type))


@ -184,12 +243,22 @@ def StripLeadingSpace(s):

 def MessageCrossRefLabel(msg_name):
  """Message cross reference label."""
-  return 'envoy_api_%s' % msg_name
+  return 'envoy_api_msg_%s' % msg_name
+

+def EnumCrossRefLabel(enum_name):
+  """Enum cross reference label."""
+  return 'envoy_api_enum_%s' % enum_name

-def FieldCrossRefLabel(msg_name, field_name):
+
+def FieldCrossRefLabel(field_name):
  """Field cross reference label."""
-  return 'envoy_api_%s_%s' % (msg_name, field_name)
+  return 'envoy_api_field_%s' % field_name
+
+
+def EnumValueCrossRefLabel(enum_value_name):
+  """Enum value cross reference label."""
+  return 'envoy_api_enum_value_%s' % enum_value_name


 def FormatAnchor(label):
@ -197,58 +266,129 @@ def FormatAnchor(label):
  return '.. _%s:\n\n' % label


-def FormatFieldAsDefinitionListItem(source_code_info, msg, path, field):
+def FormatFieldAsDefinitionListItem(type_context, field):
  """Format a FieldDescriptorProto as RST definition list item.

  Args:
-    source_code_info: SourceCodeInfo object.
-    msg: MessageDescriptorProto.
-    path: a list of path indexes as per
-      https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717.
+    type_context: contextual information for message/enum/field.
    field: FieldDescriptorProto.
  Returns:
    RST formatted definition list item.
  """
-  anchor = FormatAnchor(FieldCrossRefLabel(msg.name, field.name))
+  if field.HasField('oneof_index'):
+    oneof_comment = '\nOnly one of %s may be set.\n' % ', '.join(
+        type_context.oneof_fields[field.oneof_index])
+  else:
+    oneof_comment = ''
+  anchor = FormatAnchor(FieldCrossRefLabel(type_context.name))
  comment = '(%s) ' % FormatFieldType(
-      field) + source_code_info.LeadingCommentPathLookup(path)
+      type_context, field) + type_context.LeadingCommentPathLookup()
  return anchor + field.name + '\n' + MapLines(
-      functools.partial(Indent, 2), comment)
+      functools.partial(Indent, 2), comment + oneof_comment)


-def FormatMessageAsDefinitionList(source_code_info, path, msg):
-  """Format a MessageDescriptorProto as RST definition list.
+def FormatMessageAsDefinitionList(type_context, msg):
+  """Format a DescriptorProto as RST definition list.

  Args:
-    source_code_info: SourceCodeInfo object.
-    path: a list of path indexes as per
-      https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717.
-    msg: MessageDescriptorProto.
+    type_context: contextual information for message/enum/field.
+    msg: DescriptorProto.
  Returns:
    RST formatted definition list item.
  """
-  return '\n\n'.join(
-      FormatFieldAsDefinitionListItem(source_code_info, msg, path + [2, index],
-                                      field)
+  type_context.oneof_fields = defaultdict(list)
+  for field in msg.field:
+    if field.HasField('oneof_index'):
+      type_context.oneof_fields[field.oneof_index].append(field.name)
+  return '\n'.join(
+      FormatFieldAsDefinitionListItem(
+          type_context.Extend([2, index], field.name), field)
      for index, field in enumerate(msg.field)) + '\n'


-def FormatMessage(source_code_info, path, msg):
-  """Format a MessageDescriptorProto as RST section.
+def FormatMessage(type_context, msg):
+  """Format a DescriptorProto as RST section.

  Args:
-    source_code_info: SourceCodeInfo object.
-    path: a list of path indexes as per
-      https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717.
-    msg: MessageDescriptorProto.
+    type_context: contextual information for message/enum/field.
+    msg: DescriptorProto.
  Returns:
    RST formatted section.
  """
-  anchor = FormatAnchor(MessageCrossRefLabel(msg.name))
-  header = FormatHeader('-', msg.name)
-  comment = source_code_info.LeadingCommentPathLookup(path)
+  # Skip messages synthesized to represent map types.
+  if msg.options.map_entry:
+    return ''
+  # We need to do some extra work to recover the map type annotation from the
+  # synthesized messages.
+  type_context.map_typenames = {
+      '%s.%s' % (type_context.name, nested_msg.name): 'map<%s, %s>' % tuple(
+          map(
+              functools.partial(FormatFieldType, type_context),
+              nested_msg.field))
+      for nested_msg in msg.nested_type if nested_msg.options.map_entry
+  }
+  nested_msgs = '\n'.join(
+      FormatMessage(
+          type_context.Extend([3, index], nested_msg.name), nested_msg)
+      for index, nested_msg in enumerate(msg.nested_type))
+  nested_enums = '\n'.join(
+      FormatEnum(
+          type_context.Extend([4, index], nested_enum.name), nested_enum)
+      for index, nested_enum in enumerate(msg.enum_type))
+  anchor = FormatAnchor(MessageCrossRefLabel(type_context.name))
+  header = FormatHeader('-', type_context.name)
+  comment = type_context.LeadingCommentPathLookup()
  return anchor + header + comment + FormatMessageAsJson(
-      msg) + FormatMessageAsDefinitionList(source_code_info, path, msg)
+      type_context, msg) + FormatMessageAsDefinitionList(
+          type_context, msg) + nested_msgs + '\n' + nested_enums
+
+
+def FormatEnumValueAsDefinitionListItem(type_context, enum_value):
+  """Format a EnumValueDescriptorProto as RST definition list item.
+
+  Args:
+    type_context: contextual information for message/enum/field.
+    enum_value: EnumValueDescriptorProto.
+  Returns:
+    RST formatted definition list item.
+  """
+  anchor = FormatAnchor(EnumValueCrossRefLabel(type_context.name))
+  default_comment = '*(DEFAULT)* ' if enum_value.number == 0 else ''
+  comment = default_comment + UNICODE_INVISIBLE_SEPARATOR + type_context.LeadingCommentPathLookup(
+  )
+  return anchor + enum_value.name + '\n' + MapLines(
+      functools.partial(Indent, 2), comment)
+
+
+def FormatEnumAsDefinitionList(type_context, enum):
+  """Format a EnumDescriptorProto as RST definition list.
+
+  Args:
+    type_context: contextual information for message/enum/field.
+    enum: DescriptorProto.
+  Returns:
+    RST formatted definition list item.
+  """
+  return '\n'.join(
+      FormatEnumValueAsDefinitionListItem(
+          type_context.Extend([2, index], enum_value.name), enum_value)
+      for index, enum_value in enumerate(enum.value)) + '\n'
+
+
+def FormatEnum(type_context, enum):
+  """Format an EnumDescriptorProto as RST section.
+
+  Args:
+    type_context: contextual information for message/enum/field.
+    enum: EnumDescriptorProto.
+  Returns:
+    RST formatted section.
+  """
+  anchor = FormatAnchor(EnumCrossRefLabel(type_context.name))
+  header = FormatHeader('-', 'Enum %s' % type_context.name)
+  comment = type_context.LeadingCommentPathLookup()
+  return anchor + header + comment + FormatEnumAsDefinitionList(
+      type_context, enum)


 def FormatProtoAsBlockComment(proto):
@ -261,18 +401,19 @@ def FormatProtoAsBlockComment(proto):


 def GenerateRst(proto_file):
-  """Generate a RST representation from a FileDescriptor proto.
-
-  """
+  """Generate a RST representation from a FileDescriptor proto."""
  header = FormatHeader('=', proto_file.name)
  source_code_info = SourceCodeInfo(proto_file.source_code_info)
  # Find the earliest detached comment, attribute it to file level.
  comment = source_code_info.file_level_comment
  msgs = '\n'.join(
-      FormatMessage(source_code_info, [4, index], msg)
+      FormatMessage(TypeContext(source_code_info, [4, index], msg.name), msg)
      for index, msg in enumerate(proto_file.message_type))
+  enums = '\n'.join(
+      FormatEnum(TypeContext(source_code_info, [5, index], enum.name), enum)
+      for index, enum in enumerate(proto_file.enum_type))
  #debug_proto = FormatProtoAsBlockComment(proto_file.source_code_info)
-  return header + comment + msgs  #+ debug_proto
+  return header + comment + msgs + enums  #+ debug_proto


 if __name__ == '__main__':