From 781a02f3d8a4ae7955d9a6d39f838edb53b99745 Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 22 Nov 2017 16:31:00 -0500 Subject: [PATCH] docs: xDS overview doc, protodoc.py improvements. (#258) This patch adds an overview page introduced the v2 API concepts via a worked example. Brought in the entire transitive dep set of protos from bootstrap.proto, none of these have been cleaned up beyond the minimum required to have them build under Sphinx. Also added the ability to link to the underlying proto in messages/enums from protodoc.py generated RST. Signed-off-by: Harvey Tuch --- api/bootstrap.proto | 1 + api/cds.proto | 16 +- api/discovery.proto | 1 + api/eds.proto | 17 +- api/health_check.proto | 2 + api/lds.proto | 4 +- api/protocol.proto | 2 + api/rds.proto | 1 + api/sds.proto | 2 + docs/build.sh | 8 + docs/root/api-v2/api.rst | 2 + docs/root/configuration/configuration.rst | 3 +- docs/root/configuration/overview/tracing.rst | 2 +- .../{overview.rst => v1_overview.rst} | 6 +- .../configuration/overview/v2_overview.rst | 508 ++++++++++++++++++ .../arch_overview/dynamic_configuration.rst | 28 +- docs/root/intro/arch_overview/statistics.rst | 2 +- docs/root/intro/version_history.rst | 6 +- tools/protodoc/protodoc.py | 153 +++++- 19 files changed, 704 insertions(+), 60 deletions(-) rename docs/root/configuration/overview/{overview.rst => v1_overview.rst} (98%) create mode 100644 docs/root/configuration/overview/v2_overview.rst diff --git a/api/bootstrap.proto b/api/bootstrap.proto index 081da30e..3af04473 100644 --- a/api/bootstrap.proto +++ b/api/bootstrap.proto @@ -1,3 +1,4 @@ +// [#protodoc-title: Bootstrap] // This proto is expected to be provided on disk or via the command-line to // Envoy. It provides sufficient information for Envoy to fetch the rest of // its configuration from either disk or management server(s). diff --git a/api/cds.proto b/api/cds.proto index be5e92f8..9a74ce6e 100644 --- a/api/cds.proto +++ b/api/cds.proto @@ -14,6 +14,8 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; +// [#protodoc-title: CDS] + // Return list of all clusters this proxy will load balance to. service ClusterDiscoveryService { rpc StreamClusters(stream DiscoveryRequest) @@ -127,6 +129,7 @@ message Cluster { UpstreamTlsContext tls_context = 11; oneof protocol_options { + // [#not-implemented-hide:] TcpProtocolOptions tcp_protocol_options = 12; Http1ProtocolOptions http_protocol_options = 13; // Even if default HTTP2 protocol options are desired, this field must be @@ -136,6 +139,7 @@ message Cluster { // with ALPN, http2 must be specified. As an aside this allows HTTP/2 // connections to happen over plain text. Http2ProtocolOptions http2_protocol_options = 14; + // [#not-implemented-hide:] GrpcProtocolOptions grpc_protocol_options = 15; } @@ -268,10 +272,14 @@ message Cluster { // Specifications for subsets. For each entry, LbEndpoint.Metadata's // "envoy.lb" namespace is traversed and a subset is created for each unique // combination of key and value. For example: - // { "subset_selectors": [ - // { "keys": [ "version" ] }, - // { "keys": [ "stage", "hardware_type" ] } - // ]} + // + // .. code-block:: json + // + // { "subset_selectors": [ + // { "keys": [ "version" ] }, + // { "keys": [ "stage", "hardware_type" ] } + // ]} + // // A subset is matched when the metadata from the selected route and // weighted cluster contains the same keys and values as the subset's // metadata. The same host may appear in multiple subsets. diff --git a/api/discovery.proto b/api/discovery.proto index 8a6d3b99..73532a59 100644 --- a/api/discovery.proto +++ b/api/discovery.proto @@ -54,6 +54,7 @@ message DiscoveryResponse { string version_info = 1; repeated google.protobuf.Any resources = 2; // Canary is used to support two Envoy command line flags: + // // * --terminate-on-canary-transition-failure. When set, Envoy is able to // terminate if it detects that configuration is stuck at canary. Consider // this example sequence of updates: diff --git a/api/eds.proto b/api/eds.proto index 85a7ecb3..a1fe43b2 100644 --- a/api/eds.proto +++ b/api/eds.proto @@ -10,6 +10,8 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +// [#protodoc-title: EDS] + service EndpointDiscoveryService { // The resource_names field in DiscoveryRequest specifies a list of clusters // to subscribe to updates for. @@ -159,8 +161,12 @@ message UpstreamLocalityStats { // single HTTP or gRPC request or stream is counted as one request. A TCP // connection is also treated as one request. There is no explicit // total_requests field below for a locality, but it may be inferred from: + // + // .. code-block:: none + // // total_requests = total_successful_requests + total_requests_in_progress + - // total_error_requests + // total_error_requests + // // The total number of requests successfully completed by the endpoints in the // locality. These include non-5xx responses for HTTP, where errors // originate at the client and the endpoint responded successfuly. For gRPC, @@ -188,9 +194,12 @@ message ClusterStats { // summing upstream_locality_stats. In addition, below there are additional // cluster-wide stats. The following total_requests equality holds at the // cluster-level: - // total_requests = sum_locality(total_successful_requests) + - // sum_locality(total_requests_in_progress) + - // sum_locality(total_error_requests) + total_dropped_requests + // + // .. code-block:: none + // + // sum_locality(total_successful_requests) + sum_locality(total_requests_in_progress) + + // sum_locality(total_error_requests) + total_dropped_requests` + // // The total number of dropped requests. This covers requests // deliberately dropped by the drop_overload policy and circuit breaking. uint64 total_dropped_requests = 3; diff --git a/api/health_check.proto b/api/health_check.proto index c379b298..39c24778 100644 --- a/api/health_check.proto +++ b/api/health_check.proto @@ -1,3 +1,5 @@ +// [#protodoc-title: Health check] + syntax = "proto3"; package envoy.api.v2; diff --git a/api/lds.proto b/api/lds.proto index d59473a7..62db4ddb 100644 --- a/api/lds.proto +++ b/api/lds.proto @@ -1,3 +1,4 @@ +// [#protodoc-title: LDS] // This is heavily derived from // https://lyft.github.io/envoy/docs/configuration/listeners/listeners.html // The v2 gRPC API differences are tagged with [V2-API-DIFF]. @@ -51,7 +52,7 @@ message Filter { // listener [V2-API-DIFF]. message FilterChainMatch { // If non-empty, the SNI domains to consider. May contain a wildcard prefix, - // e.g. *.example.com. + // e.g. ``*.example.com``. repeated string sni_domains = 1; // If non-empty, an IP address and prefix length to match addresses when the @@ -117,6 +118,7 @@ message Listener { // A list of filter chains to consider for this listener. The FilterChain with // the most specific FilterChainMatch criteria is used on a connection. The // algorithm works as follows: + // // 1. If SNI information is presented at connection time, only the // FilterChains matching the SNI are considered. Otherwise, only // FilterChains with no SNI domains are considered. diff --git a/api/protocol.proto b/api/protocol.proto index 4205d72b..480e3d73 100644 --- a/api/protocol.proto +++ b/api/protocol.proto @@ -1,3 +1,5 @@ +// [#protodoc-title: Protocol options] + syntax = "proto3"; package envoy.api.v2; diff --git a/api/rds.proto b/api/rds.proto index 82221e81..94e3ea9b 100644 --- a/api/rds.proto +++ b/api/rds.proto @@ -1,3 +1,4 @@ +// [#protodoc-title: RDS] // This is heavily derived from // https://lyft.github.io/envoy/docs/configuration/http_conn_man/route_config/route_config.html#config-http-conn-man-route-table. // The v2 gRPC API differences are tagged with [V2-API-DIFF]. diff --git a/api/sds.proto b/api/sds.proto index 393f9f9f..08d5f3af 100644 --- a/api/sds.proto +++ b/api/sds.proto @@ -1,3 +1,5 @@ +// [#protodoc-title: SDS] + syntax = "proto3"; package envoy.api.v2; diff --git a/docs/build.sh b/docs/build.sh index db7e6fa9..702180f6 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -28,6 +28,14 @@ bazel --batch build ${BAZEL_BUILD_OPTIONS} //api --aspects \ PROTO_RST=" /api/address/api/address.proto.rst /api/base/api/base.proto.rst + /api/bootstrap/api/bootstrap.proto.rst + /api/cds/api/cds.proto.rst + /api/discovery/api/discovery.proto.rst + /api/eds/api/eds.proto.rst + /api/health_check/api/health_check.proto.rst + /api/lds/api/lds.proto.rst + /api/rds/api/rds.proto.rst + /api/sds/api/sds.proto.rst /api/filter/accesslog/api/filter/accesslog.proto.rst /api/filter/fault/api/filter/fault.proto.rst /api/filter/http/http_connection_manager/api/filter/http/http_connection_manager.proto.rst diff --git a/docs/root/api-v2/api.rst b/docs/root/api-v2/api.rst index f42fa116..1a99b1dc 100644 --- a/docs/root/api-v2/api.rst +++ b/docs/root/api-v2/api.rst @@ -1,3 +1,5 @@ +.. _envoy_api_reference: + Envoy v2 API reference ====================== diff --git a/docs/root/configuration/configuration.rst b/docs/root/configuration/configuration.rst index 5d5fbc13..375020c0 100644 --- a/docs/root/configuration/configuration.rst +++ b/docs/root/configuration/configuration.rst @@ -7,7 +7,8 @@ Configuration reference :maxdepth: 2 :includehidden: - overview/overview + overview/v1_overview + overview/v2_overview listeners/listeners network_filters/network_filters http_conn_man/http_conn_man diff --git a/docs/root/configuration/overview/tracing.rst b/docs/root/configuration/overview/tracing.rst index 299dff4b..2d413c8e 100644 --- a/docs/root/configuration/overview/tracing.rst +++ b/docs/root/configuration/overview/tracing.rst @@ -5,7 +5,7 @@ Tracing The :ref:`tracing ` configuration specifies global settings for the HTTP tracer used by Envoy. The configuration is defined on the :ref:`server's top level configuration -`. Envoy may support other tracers in the future, but right now the HTTP tracer is +`. Envoy may support other tracers in the future, but right now the HTTP tracer is the only one supported. .. code-block:: json diff --git a/docs/root/configuration/overview/overview.rst b/docs/root/configuration/overview/v1_overview.rst similarity index 98% rename from docs/root/configuration/overview/overview.rst rename to docs/root/configuration/overview/v1_overview.rst index 05a5f9e9..7531d1a5 100644 --- a/docs/root/configuration/overview/overview.rst +++ b/docs/root/configuration/overview/v1_overview.rst @@ -1,7 +1,7 @@ -.. _config_overview: +.. _config_overview_v1: -Overview -======== +Overview (v1 API) +================= The Envoy configuration format is written in JSON and is validated against a JSON schema. The schema can be found in :repo:`source/common/json/config_schemas.cc`. The main configuration for the diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst new file mode 100644 index 00000000..58664c87 --- /dev/null +++ b/docs/root/configuration/overview/v2_overview.rst @@ -0,0 +1,508 @@ +.. _config_overview_v2: + +Overview (v2 API) +================= + +The Envoy v2 APIs are defined as `proto3 +`_ `Protocol Buffers +`_ in the `data plane API +repository `_. They evolve the +existing :ref:`v1 xDS APIs and concepts ` to support: + +* Streaming delivery of xDS API updates via gRPC. This reduces resource requirements and can + lower the update latency. +* A new REST-JSON API in which the JSON/YAML formats are derived mechanically via the `proto3 + canonical JSON mapping + `_. +* Delivery of updates via the filesystem, REST-JSON or gRPC endpoints. +* Advanced load balancing through an extended endpoint assignment API and load + and resource utilization reporting to management servers. +* `Stronger consistency and ordering properties + `_ + when needed. The v2 APIs still maintain a baseline eventual consistency model. + +See the `xDS protocol description `_ for +further details on aspects of v2 message exchange between Envoy and the management server. + +Bootstrap configuration +----------------------- + +To use the v2 API, it's necessary to supply a bootstrap configuration file. This +provides static server configuration and configures Envoy to access :ref:`dynamic +configuration if needed `. As with the v1 +JSON/YAML configuration, this is supplied on the command-line via the :option:`-c` +flag, i.e.: + +.. code-block:: console + + ./envoy -c .{json,yaml,pb,pb_text} + +where the extension reflects the underlying v2 config representation. + +The :ref:`Bootstrap ` message is the root of the +configuration. A key concept in the :ref:`Bootstrap ` +message is the distinction between static and dynamic resouces. Resources such +as a :ref:`Listener ` or :ref:`Cluster +` may be supplied either statically in +:ref:`static_resources ` or have an xDS service such as :ref:`LDS +` or :ref:`CDS ` configured in +:ref:`dynamic_resources `. + +Example +------- + +Below we will use YAML representation of the config protos and a running example +of a service proxying HTTP from 127.0.0.1:10000 to 127.0.0.2:1234. + +Static +^^^^^^ + +A minimal fully static bootstrap config is provided below: + +.. code-block:: yaml + + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: { address: 127.0.0.1, port_value: 9901 } + + static_resources: + listeners: + - name: listener_0 + address: + socket_address: { address: 127.0.0.1, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: some_service } + http_filters: + - name: envoy.router + clusters: + - name: some_service + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + hosts: [{ socket_address: { address: 127.0.0.2, port_value: 1234 }}] + +Mostly static with dynamic EDS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A bootstrap config that continues from the above example with :ref:`dynamic endpoint +discovery ` via an EDS gRPC management server listening +on 127.0.0.3:5678 is provided below: + +.. code-block:: yaml + + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: { address: 127.0.0.1, port_value: 9901 } + + static_resources: + listeners: + - name: listener_0 + address: + socket_address: { address: 127.0.0.1, port_value: 10000 } + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: some_service } + http_filters: + - name: envoy.router + clusters: + - name: some_service + connect_timeout: 0.25s + lb_policy: ROUND_ROBIN + type: EDS + eds_cluster_config: + eds_config: + api_config_source: + api_type: GRPC + cluster_name: [xds_cluster] + - name: xds_cluster + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + hosts: [{ socket_address: { address: 127.0.0.3, port_value: 5678 }}] + +Notice above that *xds_cluster* is defined to point Envoy at the management server. Even in +an otherwise completely dynamic configurations, some static resources need to be defined to point Envoy at +its xDS management server(s). + +In the above example, the EDS management server could then return a proto encoding of a +:ref:`DiscoveryResponse `: + +.. code-block:: yaml + + version_info: "0" + resources: + - "@type": type.googleapis.com/envoy.api.v2.ClusterLoadAssignment + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.2 + port_value: 1234 + + +The versioning and type URL scheme that appear above are explained in more +detail in the `streaming gRPC subscription protocol +`_ +documentation. + +Dynamic +^^^^^^^ + +A fully dynamic bootstrap configuration, in which all resources other than +those belonging to the management server are discovered via xDS is provided +below: + +.. code-block:: yaml + + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: { address: 127.0.0.1, port_value: 9901 } + + dynamic_resources: + lds_config: + api_config_source: + api_type: GRPC + cluster_name: [xds_cluster] + cds_config: + api_config_source: + api_type: GRPC + cluster_name: [xds_cluster] + + static_resources: + clusters: + - name: xds_cluster + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + hosts: [{ socket_address: { address: 127.0.0.3, port_value: 5678 }}] + +The management server could respond to LDS requests with: + +.. code-block:: yaml + + version_info: "0" + resources: + - "@type": type.googleapis.com/envoy.api.v2.Listener + name: listener_0 + address: + socket_address: + address: 127.0.0.1 + port_value: 10000 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + stat_prefix: ingress_http + codec_type: AUTO + rds: + route_config_name: local_route + config_source: + api_config_source: + api_type: GRPC + cluster_name: [xds_cluster] + http_filters: + - name: envoy.router + +The management server could respond to RDS requests with: + +.. code-block:: yaml + + version_info: "0" + resources: + - "@type": type.googleapis.com/envoy.api.v2.RouteConfiguration + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: some_service } + +The management server could respond to CDS requests with: + +.. code-block:: yaml + + version_info: "0" + resources: + - "@type": type.googleapis.com/envoy.api.v2.Cluster + name: some_service + connect_timeout: 0.25s + lb_policy: ROUND_ROBIN + type: EDS + eds_cluster_config: + eds_config: + api_config_source: + api_type: GRPC + cluster_name: [xds_cluster] + +The management server could respond to EDS requests with: + +.. code-block:: yaml + + version_info: "0" + resources: + - "@type": type.googleapis.com/envoy.api.v2.ClusterLoadAssignment + cluster_name: some_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.2 + port_value: 1234 + +Management server +----------------- + +A v2 xDS management server will implement the below endpoints as required for +gRPC and/or REST serving. In both streaming gRPC and +REST-JSON cases, a :ref:`DiscoveryRequest ` is sent and a +:ref:`DiscoveryResponse ` received following the +`xDS protocol `_. + +.. _v2_grpc_streaming_endpoints: + +gRPC streaming endpoints +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. http:post:: /envoy.api.v2.ClusterDiscoveryService/StreamClusters + +See `cds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + cds_config: + api_config_source: + api_type: GRPC + cluster_name: [some_xds_cluster] + +is set in the :ref:`dynamic_resources +` of the :ref:`Bootstrap +` config. + +.. http:post:: /envoy.api.v2.EndpointDiscoveryService/StreamEndpoints + +See `eds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + eds_config: + api_config_source: + api_type: GRPC + cluster_name: [some_xds_cluster] + +is set in the :ref:`eds_cluster_config +` field of the :ref:`Cluster +` config. + +.. http:post:: /envoy.api.v2.ListenerDiscoveryService/StreamListeners + +See `lds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + lds_config: + api_config_source: + api_type: GRPC + cluster_name: [some_xds_cluster] + +is set in the :ref:`dynamic_resources +` of the :ref:`Bootstrap +` config. + +.. http:post:: /envoy.api.v2.RouteDiscoveryService/StreamRoutes + +See `rds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + route_config_name: some_route_name + config_source: + api_config_source: + api_type: GRPC + cluster_name: [some_xds_cluster] + +is set in the :ref:`rds +` field of the :ref:`HttpConnectionManager +` config. + +REST endpoints +^^^^^^^^^^^^^^ + +.. http:post:: /v2/discovery:clusters + +See `cds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + cds_config: + api_config_source: + api_type: REST + cluster_name: [some_xds_cluster] + +is set in the :ref:`dynamic_resources +` of the :ref:`Bootstrap +` config. + +.. http:post:: /v2/discovery:endpoints + +See `eds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + eds_config: + api_config_source: + api_type: REST + cluster_name: [some_xds_cluster] + +is set in the :ref:`eds_cluster_config +` field of the :ref:`Cluster +` config. + +.. http:post:: /v2/discovery:listeners + +See `lds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + lds_config: + api_config_source: + api_type: REST + cluster_name: [some_xds_cluster] + +is set in the :ref:`dynamic_resources +` of the :ref:`Bootstrap +` config. + +.. http:post:: /v2/discovery:routes + +See `rds.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + route_config_name: some_route_name + config_source: + api_config_source: + api_type: REST + cluster_name: [some_xds_cluster] + +is set in the :ref:`rds +` field of the :ref:`HttpConnectionManager +` config. + +Aggregated Discovery Service +---------------------------- + +While fundamentally Envoy employs an eventual consistency model, ADS provides an +opportunity to sequence API update pushes and ensure affinity of a single +management server for an Envoy node for API updates. ADS allows one or more APIs +to be delivered on a single gRPC bidi stream by the management server, and +within an API to have all resources aggregated onto a single stream. Without +this, some APIs such as RDS and EDS may require the management of multiple +streams and connections to distinct management servers. + +ADS will allow for hitless updates of configuration by appropriate sequencing. +For example, suppose *foo.com* was mappped to cluster *X*. We wish to change the +mapping in the route table to point *foo.com* at cluster *Y*. In order to do +this, a CDS/EDS update must first be delivered containing both clusters *X* and +*Y*. + +Without ADS, the CDS/EDS/RDS streams may point at distinct management servers, +or when on the same management server at distinct gRPC streams/connections that +require coordination. The EDS resource requests may be split across two distinct +streams, one for *X* and one for *Y*. ADS allows these to be coalesced to a +single stream to a single management server, avoiding the need for distributed +synchronization to correctly sequence the update. With ADS, the management +server would deliver the CDS, EDS and then RDS updates on a single stream. + +ADS is only available for gRPC streaming (not REST) and is described more fully +in `this +`_ +document. The gRPC endpoint is: + +.. http:post:: /envoy.api.v2.AggregatedDiscoveryService/StreamAggregatedResources + +See `discovery.proto +`_ +for the service definition. This is used by Envoy as a client when + +.. code-block:: yaml + + ads_config: + api_type: GRPC + cluster_name: [some_ads_cluster] + +is set in the :ref:`dynamic_resources +` of the :ref:`Bootstrap +` config. + +When this is set, any of the configuration sources :ref:`above ` can +be set to use the ADS channel. For example, a LDS config could be changed from + +.. code-block:: yaml + + lds_config: + api_config_source: + api_type: REST + cluster_name: [some_xds_cluster] + +to + +.. code-block:: yaml + + lds_config: {ads: {}} + +with the effect that the LDS stream will be directed to *some_ads_cluster* over +the shared ADS channel. + +Status +------ + +The current API status is tracked `here +`_. All features described +in the :ref:`v2 API reference ` are implemented unless +otherwise noted. diff --git a/docs/root/intro/arch_overview/dynamic_configuration.rst b/docs/root/intro/arch_overview/dynamic_configuration.rst index f9762e80..64425cf6 100644 --- a/docs/root/intro/arch_overview/dynamic_configuration.rst +++ b/docs/root/intro/arch_overview/dynamic_configuration.rst @@ -12,6 +12,7 @@ of the options currently available. * Top level configuration :ref:`reference `. * :ref:`Reference configurations `. +* Envoy :ref:`v2 API overview `. Fully static ------------ @@ -28,19 +29,20 @@ graceful hot restarts. .. _arch_overview_dynamic_config_sds: -SDS only --------- +SDS/EDS only +------------ The :ref:`service discovery service (SDS) API ` provides a more advanced -mechanism by which Envoy can discover members of an upstream cluster. Layered on top of a static +mechanism by which Envoy can discover members of an upstream cluster. SDS has been renamed to Endpoint +Discovery Service (EDS) in the :ref:`v2 API `. Layered on top of a static configuration, SDS allows an Envoy deployment to circumvent the limitations of DNS (maximum records in a response, etc.) as well as consume more information used in load balancing and routing (e.g., canary status, zone, etc.). .. _arch_overview_dynamic_config_cds: -SDS and CDS ------------ +SDS/EDS and CDS +--------------- The :ref:`cluster discovery service (CDS) API ` layers on a mechanism by which Envoy can discover upstream clusters used during routing. Envoy will gracefully add, update, @@ -50,28 +52,28 @@ Typically, when doing HTTP routing along with CDS (but without route discovery s implementors will make use of the router's ability to forward requests to a cluster specified in an :ref:`HTTP request header `. -Although it is possible to use CDS without SDS by specifying fully static clusters, we recommend -still using the SDS API for clusters specified via CDS. Internally, when a cluster definition is +Although it is possible to use CDS without SDS/EDS by specifying fully static clusters, we recommend +still using the SDS/EDS API for clusters specified via CDS. Internally, when a cluster definition is updated, the operation is graceful. However, all existing connection pools will be drained and -reconnected. SDS does not suffer from this limitation. When hosts are added and removed via SDS, +reconnected. SDS/EDS does not suffer from this limitation. When hosts are added and removed via SDS/EDS, the existing hosts in the cluster are unaffected. .. _arch_overview_dynamic_config_rds: -SDS, CDS, and RDS ------------------ +SDS/EDS, CDS, and RDS +--------------------- The :ref:`route discovery service (RDS) API ` layers on a mechanism by which Envoy can discover the entire route configuration for an HTTP connection manager filter at runtime. The route configuration will be gracefully swapped in without affecting existing requests. This API, -when used alongside SDS and CDS, allows implementors to build a complex routing topology +when used alongside SDS/EDS and CDS, allows implementors to build a complex routing topology (:ref:`traffic shifting `, blue/green deployment, etc.) that will not require any Envoy restarts other than to obtain a new Envoy binary. .. _arch_overview_dynamic_config_lds: -SDS, CDS, RDS, and LDS ----------------------- +SDS/EDS, CDS, RDS, and LDS +-------------------------- The :ref:`listener discovery service (LDS) ` layers on a mechanism by which Envoy can discover entire listeners at runtime. This includes all filter stacks, up to and including diff --git a/docs/root/intro/arch_overview/statistics.rst b/docs/root/intro/arch_overview/statistics.rst index 38325f78..be9b1811 100644 --- a/docs/root/intro/arch_overview/statistics.rst +++ b/docs/root/intro/arch_overview/statistics.rst @@ -22,4 +22,4 @@ batched and periodically flushed to improve performance. Histograms are written received. Note: what were previously referred to as timers have become histograms as the only difference between the two representations was the units. -Statistics :ref:`configuration `. +Statistics :ref:`configuration `. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b55fd75b..b6528a60 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -6,7 +6,7 @@ Version history * macOS is :repo:`now supported `. (A few features are missing such as hot restart and original destination routing). -* YAML is now directly supported for :ref:`config files `. +* YAML is now directly supported for :ref:`config files `. * Added :ref:`/routes ` admin endpoint. * End-to-end flow control is now supported for TCP proxy, HTTP/1, and HTTP/2. HTTP flow control that includes filter buffering is incomplete and will be implemented in 1.5.0. @@ -103,7 +103,7 @@ Version history * The :ref:`MongoDB filter ` now emits logs that are fully valid JSON. * The CPU profiler output path is now :ref:`configurable `. -* A :ref:`watchdog system ` has been added that can kill the server if a deadlock +* A :ref:`watchdog system ` has been added that can kill the server if a deadlock is detected. * A :ref:`route table checking tool ` has been added that can be used to test route tables before use. @@ -124,7 +124,7 @@ Version history * :ref:`Cluster discovery service (CDS) API `. * :ref:`Outlier detection ` (passive health checking). -* Envoy configuration is now checked against a :ref:`JSON schema `. +* Envoy configuration is now checked against a :ref:`JSON schema `. * :ref:`Ring hash ` consistent load balancer, as well as HTTP consistent hash routing :ref:`based on a policy `. * Vastly :ref:`enhanced global rate limit configuration ` via the HTTP diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index cccaf722..1816cb86 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -43,10 +43,14 @@ V2_API_DIFF_ANNOTATION = 'v2-api-diff' VALID_ANNOTATIONS = set([ DOC_TITLE_ANNOTATION, NOT_IMPLEMENTED_WARN_ANNOTATION, - NOT_IMPLEMENTED_HIDE_ANNOTATION, V2_API_DIFF_ANNOTATION, - COMMENT_ANNOTATION + NOT_IMPLEMENTED_HIDE_ANNOTATION, V2_API_DIFF_ANNOTATION, COMMENT_ANNOTATION ]) +# Template for data-plane-api URLs. +# TODO(htuch): Add the ability to build a permalink by feeding a hash +# to the tool or inferring from local tree (only really make sense in CI). +DATA_PLANE_API_URL_FMT = 'https://github.com/envoyproxy/data-plane-api/blob/master/%s#L%d' + class ProtodocError(Exception): """Base error class for the protodoc module.""" @@ -87,7 +91,8 @@ def ExtractAnnotations(s): class SourceCodeInfo(object): """Wrapper for SourceCodeInfo proto.""" - def __init__(self, source_code_info): + def __init__(self, name, source_code_info): + self._name = name self._proto = source_code_info @property @@ -120,6 +125,20 @@ class SourceCodeInfo(object): StripLeadingSpace(location.leading_comments) + '\n') return '', [] + def GithubUrl(self, path): + """Obtain data-plane-api Github URL by path from SourceCodeInfo. + + Args: + path: a list of path indexes as per + https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. + Returns: + A string with a corresponding data-plan-api GitHub Url. + """ + for location in self._proto.location: + if location.path == path: + return DATA_PLANE_API_URL_FMT % (self._name, location.span[0]) + return '' + class TypeContext(object): """Contextual information for a message/field. @@ -128,14 +147,14 @@ class TypeContext(object): nested messages/enums. """ - def __init__(self, source_code_info, path, name): + def __init__(self, source_code_info, name): # SourceCodeInfo as per # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto. self.source_code_info = source_code_info # path: a list of path indexes as per # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. # Extended as nested objects are traversed. - self.path = path + self.path = [] # Message/enum/field name. Extended as nested objects are traversed. self.name = name # Map from type name to the correct type annotation string, e.g. from @@ -147,15 +166,75 @@ class TypeContext(object): # Map from a message's oneof index to the "required" bool property. self.oneof_required = {} - def Extend(self, path, name): + def _Extend(self, path, name): extended = copy.deepcopy(self) extended.path.extend(path) - extended.name = '%s.%s' % (self.name, name) + if not self.name: + extended.name = name + else: + extended.name = '%s.%s' % (self.name, name) return extended + def ExtendMessage(self, index, name): + """Extend type context with a message. + + Args: + index: message index in file. + name: message name. + """ + return self._Extend([4, index], name) + + def ExtendNestedMessage(self, index, name): + """Extend type context with a nested message. + + Args: + index: nested message index in message. + name: message name. + """ + return self._Extend([3, index], name) + + def ExtendField(self, index, name): + """Extend type context with a field. + + Args: + index: field index in message. + name: field name. + """ + return self._Extend([2, index], name) + + def ExtendEnum(self, index, name): + """Extend type context with an enum. + + Args: + index: enum index in file. + name: enum name. + """ + return self._Extend([5, index], name) + + def ExtendNestedEnum(self, index, name): + """Extend type context with a nested enum. + + Args: + index: enum index in message. + name: enum name. + """ + return self._Extend([4, index], name) + + def ExtendEnumValue(self, index, name): + """Extend type context with an enum enum. + + Args: + index: enum value index in enum. + name: value name. + """ + return self._Extend([2, index], name) + def LeadingCommentPathLookup(self): return self.source_code_info.LeadingCommentPathLookup(self.path) + def GithubUrl(self): + return self.source_code_info.GithubUrl(self.path) + def MapLines(f, s): """Apply a function across each line in a flat string. @@ -210,11 +289,12 @@ def FormatHeaderFromFile(style, file_level_comment, alt): Returns: RST formatted header, and file level comment without page title strings. """ + anchor = FormatAnchor(FileCrossRefLabel(alt)) stripped_comment, annotations = ExtractAnnotations(file_level_comment) if DOC_TITLE_ANNOTATION in annotations: - return FormatHeader(style, - annotations[DOC_TITLE_ANNOTATION]), stripped_comment - return FormatHeader(style, alt), stripped_comment + return anchor + FormatHeader( + style, annotations[DOC_TITLE_ANNOTATION]), stripped_comment + return anchor + FormatHeader(style, alt), stripped_comment def FormatFieldTypeAsJson(type_context, field): @@ -246,11 +326,13 @@ def FormatMessageAsJson(type_context, msg): """ lines = [] for index, field in enumerate(msg.field): - field_type_context = type_context.Extend([2, index], field.name) - leading_comment, comment_annotations = field_type_context.LeadingCommentPathLookup() + field_type_context = type_context.ExtendField(index, field.name) + leading_comment, comment_annotations = field_type_context.LeadingCommentPathLookup( + ) if NOT_IMPLEMENTED_HIDE_ANNOTATION in comment_annotations: continue - lines.append('"%s": %s' % (field.name, FormatFieldTypeAsJson(type_context, field))) + lines.append('"%s": %s' % (field.name, + FormatFieldTypeAsJson(type_context, field))) return '.. code-block:: json\n\n {\n' + ',\n'.join(IndentLines( 4, lines)) + '\n }\n\n' @@ -334,6 +416,11 @@ def StripLeadingSpace(s): return MapLines(lambda s: s[1:], s) +def FileCrossRefLabel(msg_name): + """File cross reference label.""" + return 'envoy_api_file_%s' % msg_name + + def MessageCrossRefLabel(msg_name): """Message cross reference label.""" return 'envoy_api_msg_%s' % msg_name @@ -374,7 +461,7 @@ def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field): field.oneof_index] else '\nOnly one of %s may be set.\n' oneof_comment = oneof_template % ', '.join( FormatInternalLink( - f, FieldCrossRefLabel(outer_type_context.Extend([], f).name)) + f, FieldCrossRefLabel(outer_type_context.ExtendField(0, f).name)) for f in type_context.oneof_fields[field.oneof_index]) else: oneof_comment = '' @@ -406,8 +493,12 @@ def FormatMessageAsDefinitionList(type_context, msg): """ type_context.oneof_fields = defaultdict(list) type_context.oneof_required = defaultdict(bool) - for field in msg.field: + for index, field in enumerate(msg.field): if field.HasField('oneof_index'): + _, comment_annotations = type_context.ExtendField( + index, field.name).LeadingCommentPathLookup() + if NOT_IMPLEMENTED_HIDE_ANNOTATION in comment_annotations: + continue type_context.oneof_fields[field.oneof_index].append(field.name) for index, oneof_decl in enumerate(msg.oneof_decl): if oneof_decl.options.HasExtension(validate_pb2.required): @@ -415,7 +506,7 @@ def FormatMessageAsDefinitionList(type_context, msg): validate_pb2.required] return '\n'.join( FormatFieldAsDefinitionListItem( - type_context, type_context.Extend([2, index], field.name), field) + type_context, type_context.ExtendField(index, field.name), field) for index, field in enumerate(msg.field)) + '\n' @@ -443,18 +534,20 @@ def FormatMessage(type_context, msg): } nested_msgs = '\n'.join( FormatMessage( - type_context.Extend([3, index], nested_msg.name), nested_msg) + type_context.ExtendNestedMessage(index, nested_msg.name), nested_msg) for index, nested_msg in enumerate(msg.nested_type)) nested_enums = '\n'.join( FormatEnum( - type_context.Extend([4, index], nested_enum.name), nested_enum) + type_context.ExtendNestedEnum(index, nested_enum.name), nested_enum) for index, nested_enum in enumerate(msg.enum_type)) anchor = FormatAnchor(MessageCrossRefLabel(type_context.name)) header = FormatHeader('-', type_context.name) + proto_link = FormatExternalLink('[%s proto]' % type_context.name, + type_context.GithubUrl()) + '\n\n' leading_comment, annotations = type_context.LeadingCommentPathLookup() if NOT_IMPLEMENTED_HIDE_ANNOTATION in annotations: return '' - return anchor + header + leading_comment + FormatMessageAsJson( + return anchor + header + proto_link + leading_comment + FormatMessageAsJson( type_context, msg) + FormatMessageAsDefinitionList( type_context, msg) + nested_msgs + '\n' + nested_enums @@ -489,7 +582,7 @@ def FormatEnumAsDefinitionList(type_context, enum): """ return '\n'.join( FormatEnumValueAsDefinitionListItem( - type_context.Extend([2, index], enum_value.name), enum_value) + type_context.ExtendEnumValue(index, enum_value.name), enum_value) for index, enum_value in enumerate(enum.value)) + '\n' @@ -504,10 +597,12 @@ def FormatEnum(type_context, enum): """ anchor = FormatAnchor(EnumCrossRefLabel(type_context.name)) header = FormatHeader('-', 'Enum %s' % type_context.name) + proto_link = FormatExternalLink('[%s proto]' % type_context.name, + type_context.GithubUrl()) + '\n\n' leading_comment, annotations = type_context.LeadingCommentPathLookup() if NOT_IMPLEMENTED_HIDE_ANNOTATION in annotations: return '' - return anchor + header + leading_comment + FormatEnumAsDefinitionList( + return anchor + header + proto_link + leading_comment + FormatEnumAsDefinitionList( type_context, enum) @@ -522,20 +617,20 @@ def FormatProtoAsBlockComment(proto): def GenerateRst(proto_file): """Generate a RST representation from a FileDescriptor proto.""" - source_code_info = SourceCodeInfo(proto_file.source_code_info) + source_code_info = SourceCodeInfo(proto_file.name, + proto_file.source_code_info) # Find the earliest detached comment, attribute it to file level. # Also extract file level titles if any. header, comment = FormatHeaderFromFile( '=', source_code_info.file_level_comment, proto_file.name) - package_prefix = NormalizeFQN('.' + proto_file.package + '.') + package_prefix = NormalizeFQN('.' + proto_file.package + '.')[:-1] + package_type_context = TypeContext(source_code_info, package_prefix) msgs = '\n'.join( - FormatMessage( - TypeContext(source_code_info, [4, index], package_prefix + msg.name), - msg) for index, msg in enumerate(proto_file.message_type)) + FormatMessage(package_type_context.ExtendMessage(index, msg.name), msg) + for index, msg in enumerate(proto_file.message_type)) enums = '\n'.join( - FormatEnum( - TypeContext(source_code_info, [5, index], package_prefix + enum.name), - enum) for index, enum in enumerate(proto_file.enum_type)) + FormatEnum(package_type_context.ExtendEnum(index, enum.name), enum) + for index, enum in enumerate(proto_file.enum_type)) debug_proto = FormatProtoAsBlockComment(proto_file) return header + comment + msgs + enums # + debug_proto