From b796da4964af6c332bd7d29ae4f08d0a98bf5352 Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 9 Jan 2018 10:03:19 -0500 Subject: [PATCH] api: Google gRPC client library configuration. (#398) In support of https://github.com/envoyproxy/envoy/issues/2200 and some Google internal needs, we are planning on adding support to Envoy to allow a configuration (or possibly build) driven decision on whether to using the existing Envoy in-built Grpc::AsyncClient or the Google C++ gRPC client library (https://grpc.io/grpc/cpp/index.html). To move in this direction, the idea is we have the xDS ApiConfigSources, rate limit service config and other filter configurations point at a GrpcService object. This can be configured to use an Envoy cluster, where Grpc::AsyncClient will orchestrate communication, or to contain the config needed to establish a channel in Google C++ gRPC client library. Signed-off-by: Harvey Tuch --- api/BUILD | 50 +++++++++++- api/base.proto | 56 ------------- api/bootstrap.proto | 22 ++++- api/cds.proto | 1 + api/config_source.proto | 79 ++++++++++++++++++ api/filter/accesslog/BUILD | 2 +- api/filter/accesslog/accesslog.proto | 6 +- api/filter/http/BUILD | 2 +- api/filter/http/ext_authz.proto | 4 +- api/filter/network/BUILD | 3 +- api/filter/network/ext_authz.proto | 4 +- .../network/http_connection_manager.proto | 1 + api/grpc_cluster.proto | 19 ----- api/grpc_service.proto | 81 +++++++++++++++++++ api/metrics_service.proto | 4 +- api/sds.proto | 1 + docs/build.sh | 2 + docs/root/api-v2/api.rst | 2 + docs/root/intro/arch_overview/grpc.rst | 42 ++++++++++ 19 files changed, 287 insertions(+), 94 deletions(-) create mode 100644 api/config_source.proto delete mode 100644 api/grpc_cluster.proto create mode 100644 api/grpc_service.proto diff --git a/api/BUILD b/api/BUILD index ffab4800..fcaf98d2 100644 --- a/api/BUILD +++ b/api/BUILD @@ -48,6 +48,8 @@ api_proto_library( ":address", ":base", ":cds", + ":config_source", + ":grpc_service", ":lds", ":sds", ":stats", @@ -64,6 +66,8 @@ go_proto_library( ":address_go_proto", ":base_go_proto", ":cds_go_grpc", + ":config_source_go_proto", + ":grpc_service_go_proto", ":lds_go_grpc", ":sds_go_grpc", ":stats_go_proto", @@ -100,6 +104,7 @@ api_proto_library( deps = [ ":address", ":base", + ":config_source", ":discovery", ":health_check", ":protocol", @@ -115,6 +120,7 @@ go_grpc_library( deps = [ ":address_go_proto", ":base_go_proto", + ":config_source_go_proto", ":discovery_go_grpc", ":health_check_go_proto", ":protocol_go_proto", @@ -128,6 +134,29 @@ go_grpc_library( ], ) +api_proto_library( + name = "config_source", + srcs = ["config_source.proto"], + deps = [ + ":base", + ":grpc_service", + ], +) + +go_proto_library( + name = "config_source_go_proto", + importpath = "github.com/envoyproxy/data-plane-api/api/config_source", + proto = ":config_source", + visibility = ["//visibility:public"], + deps = [ + ":base_go_proto", + ":grpc_service_go_proto", + "@com_github_gogo_protobuf//:gogo_proto_go", + "@com_github_golang_protobuf//ptypes/duration:go_default_library", + "@com_lyft_protoc_gen_validate//validate:go_default_library", + ], +) + api_proto_library( name = "discovery", srcs = ["discovery.proto"], @@ -176,8 +205,21 @@ go_grpc_library( ) api_proto_library( - name = "grpc_cluster", - srcs = ["grpc_cluster.proto"], + name = "grpc_service", + srcs = ["grpc_service.proto"], + deps = [":base"], +) + +go_proto_library( + name = "grpc_service_go_proto", + importpath = "github.com/envoyproxy/data-plane-api/api/grpc_service", + proto = ":grpc_service", + visibility = ["//visibility:public"], + deps = [ + ":base_go_proto", + "@com_github_golang_protobuf//ptypes/duration:go_default_library", + "@com_lyft_protoc_gen_validate//validate:go_default_library", + ], ) api_proto_library( @@ -239,7 +281,7 @@ api_proto_library( require_py = 0, deps = [ ":base", - ":grpc_cluster", + ":grpc_service", "@promotheus_metrics_model//:client_model", ], ) @@ -310,6 +352,7 @@ api_proto_library( has_services = 1, deps = [ ":base", + ":config_source", ":discovery", ], ) @@ -321,6 +364,7 @@ go_grpc_library( visibility = ["//visibility:public"], deps = [ ":base_go_proto", + ":config_source_go_proto", ":discovery_go_grpc", "@com_github_golang_protobuf//ptypes/wrappers:go_default_library", "@com_lyft_protoc_gen_validate//validate:go_default_library", diff --git a/api/base.proto b/api/base.proto index 7f6e6795..78e4130d 100644 --- a/api/base.proto +++ b/api/base.proto @@ -173,62 +173,6 @@ message DataSource { } } -// API configuration source. This identifies the API type and cluster that Envoy -// will use to fetch an xDS API. -message ApiConfigSource { - // APIs may be fetched via either REST or gRPC. - enum ApiType { - // REST-JSON legacy corresponds to the v1 API. - REST_LEGACY = 0; - // REST-JSON v2 API. The `canonical JSON encoding - // `_ for - // the v2 protos is used. - REST = 1; - // gRPC v2 API. - GRPC = 2; - } - ApiType api_type = 1 [(validate.rules).enum.defined_only = true]; - // Multiple cluster names may be provided. If > 1 cluster is defined, clusters - // will be cycled through if any kind of failure occurs. - // - // .. note:: - // - // The cluster with name ``cluster_name`` must be statically defined and its - // type must not be ``EDS``. - repeated string cluster_name = 2 [(validate.rules).repeated .min_items = 1]; - // For REST APIs, the delay between successive polls. - google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; -} - -// Aggregated Discovery Service (ADS) options. This is currently empty, but when -// set in :ref:`ConfigSource ` can be used to -// specify that ADS is to be used. -message AggregatedConfigSource { -} - -// Configuration for :ref:`listeners `, :ref:`clusters -// `, :ref:`routes -// `, :ref:`endpoints -// ` etc. may either be sourced from the -// filesystem or from an xDS API source. Filesystem configs are watched with -// inotify for updates. -message ConfigSource { - oneof config_source_specifier { - option (validate.required) = true; - // Path on the filesystem to source and watch for configuration updates. - // - // .. note:: - // - // The path to the source must exist at config load time. - string path = 1; - // API configuration source. - ApiConfigSource api_config_source = 2; - // When set, ADS will be used to fetch resources. The ADS API configuration - // source in the bootstrap configuration is used. - AggregatedConfigSource ads = 3; - } -} - // Configuration for transport socket in :ref:`listeners ` and // :ref:`clusters `. If the configuration is // empty, a default transport socket implementation and configuration will be diff --git a/api/bootstrap.proto b/api/bootstrap.proto index 6eb98842..ad87f43a 100644 --- a/api/bootstrap.proto +++ b/api/bootstrap.proto @@ -9,7 +9,9 @@ package envoy.api.v2; import "api/address.proto"; import "api/base.proto"; +import "api/config_source.proto"; import "api/cds.proto"; +import "api/grpc_service.proto"; import "api/lds.proto"; import "api/sds.proto"; import "api/stats.proto"; @@ -208,8 +210,20 @@ message Runtime { // Rate limit :ref:`configuration overview `. message RateLimitServiceConfig { - // Specifies the cluster manager cluster name that hosts the rate limit - // service. The client will connect to this cluster when it needs to make rate - // limit service requests. - string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; + oneof service_specifier { + option (validate.required) = true; + + // Specifies the cluster manager cluster name that hosts the rate limit + // service. The client will connect to this cluster when it needs to make + // rate limit service requests. This field is deprecated and `grpc_service` + // should be used instead. The :ref:`Envoy gRPC client + // ` will be used when this field is + // specified. + string cluster_name = 1 [(validate.rules).string.min_bytes = 1, deprecated = true]; + + // Specifies the gRPC service that hosts the rate limit service. The client + // will connect to this cluster when it needs to make rate limit service + // requests. + GrpcService grpc_service = 2; + } } diff --git a/api/cds.proto b/api/cds.proto index 86a2906c..ed530b04 100644 --- a/api/cds.proto +++ b/api/cds.proto @@ -4,6 +4,7 @@ package envoy.api.v2; import "api/address.proto"; import "api/base.proto"; +import "api/config_source.proto"; import "api/discovery.proto"; import "api/health_check.proto"; import "api/protocol.proto"; diff --git a/api/config_source.proto b/api/config_source.proto new file mode 100644 index 00000000..d8d6b405 --- /dev/null +++ b/api/config_source.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; + +package envoy.api.v2; + +import "api/grpc_service.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Configuration sources] + +// API configuration source. This identifies the API type and cluster that Envoy +// will use to fetch an xDS API. +message ApiConfigSource { + // APIs may be fetched via either REST or gRPC. + enum ApiType { + // REST-JSON legacy corresponds to the v1 API. + REST_LEGACY = 0; + // REST-JSON v2 API. The `canonical JSON encoding + // `_ for + // the v2 protos is used. + REST = 1; + // gRPC v2 API. + GRPC = 2; + } + ApiType api_type = 1 [(validate.rules).enum.defined_only = true]; + // Multiple cluster names may be provided for REST_LEGACY/REST. If > 1 + // cluster is defined, clusters will be cycled through if any kind of failure + // occurs. + // + // .. note:: + // + // The cluster with name ``cluster_name`` must be statically defined and its + // type must not be ``EDS``. + repeated string cluster_names = 2; + + // Multiple gRPC services be provided for GRPC. If > 1 cluster is defined, + // services will be cycled through if any kind of failure occurs. + // + // .. note:: + // + // If a gRPC service points to a ``cluster_name``, it must be statically + // defined and its type must not be ``EDS``. + repeated GrpcService grpc_services = 4; + + // For REST APIs, the delay between successive polls. + google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; +} + +// Aggregated Discovery Service (ADS) options. This is currently empty, but when +// set in :ref:`ConfigSource ` can be used to +// specify that ADS is to be used. +message AggregatedConfigSource { +} + +// Configuration for :ref:`listeners `, :ref:`clusters +// `, :ref:`routes +// `, :ref:`endpoints +// ` etc. may either be sourced from the +// filesystem or from an xDS API source. Filesystem configs are watched with +// inotify for updates. +message ConfigSource { + oneof config_source_specifier { + option (validate.required) = true; + // Path on the filesystem to source and watch for configuration updates. + // + // .. note:: + // + // The path to the source must exist at config load time. + string path = 1; + // API configuration source. + ApiConfigSource api_config_source = 2; + // When set, ADS will be used to fetch resources. The ADS API configuration + // source in the bootstrap configuration is used. + AggregatedConfigSource ads = 3; + } +} diff --git a/api/filter/accesslog/BUILD b/api/filter/accesslog/BUILD index bcdfd59d..e15f36a1 100644 --- a/api/filter/accesslog/BUILD +++ b/api/filter/accesslog/BUILD @@ -7,6 +7,6 @@ api_proto_library( deps = [ "//api:address", "//api:base", - "//api:grpc_cluster", + "//api:grpc_service", ], ) diff --git a/api/filter/accesslog/accesslog.proto b/api/filter/accesslog/accesslog.proto index 16cef405..f21b25b1 100644 --- a/api/filter/accesslog/accesslog.proto +++ b/api/filter/accesslog/accesslog.proto @@ -5,7 +5,7 @@ option go_package = "accesslog"; import "api/address.proto"; import "api/base.proto"; -import "api/grpc_cluster.proto"; +import "api/grpc_service.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; @@ -432,8 +432,8 @@ message CommonGrpcAccessLogConfig { // same Envoy. string log_name = 1 [(validate.rules).string.min_bytes = 1]; - // The upstream gRPC cluster that hosts the access log service. - GrpcCluster cluster = 2 [(validate.rules).message.required = true]; + // The gRPC service for the access log service. + GrpcService grpc_service = 2 [(validate.rules).message.required = true]; } // [#proto-status: experimental] diff --git a/api/filter/http/BUILD b/api/filter/http/BUILD index 4d052537..7491c013 100644 --- a/api/filter/http/BUILD +++ b/api/filter/http/BUILD @@ -61,5 +61,5 @@ api_proto_library( api_proto_library( name = "ext_authz", srcs = ["ext_authz.proto"], - deps = ["//api:grpc_cluster"], + deps = ["//api:grpc_service"], ) diff --git a/api/filter/http/ext_authz.proto b/api/filter/http/ext_authz.proto index 99bc9049..807127db 100644 --- a/api/filter/http/ext_authz.proto +++ b/api/filter/http/ext_authz.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter.http; -import "api/grpc_cluster.proto"; +import "api/grpc_service.proto"; import "validate/validate.proto"; @@ -13,7 +13,7 @@ import "validate/validate.proto"; message ExtAuthz { // The external authorization gRPC service configuration. - GrpcCluster grpc_cluster = 1; + GrpcService grpc_service = 1; // The filter's behaviour in case the external authorization service does // not respond back. If set to true then in case of failure to get a diff --git a/api/filter/network/BUILD b/api/filter/network/BUILD index 89e48af3..e87026ce 100644 --- a/api/filter/network/BUILD +++ b/api/filter/network/BUILD @@ -7,6 +7,7 @@ api_proto_library( srcs = ["http_connection_manager.proto"], deps = [ "//api:base", + "//api:config_source", "//api:protocol", "//api:rds", "//api/filter/accesslog", @@ -48,5 +49,5 @@ api_proto_library( api_proto_library( name = "ext_authz", srcs = ["ext_authz.proto"], - deps = ["//api:grpc_cluster"], + deps = ["//api:grpc_service"], ) diff --git a/api/filter/network/ext_authz.proto b/api/filter/network/ext_authz.proto index 64e5a393..a256caaa 100644 --- a/api/filter/network/ext_authz.proto +++ b/api/filter/network/ext_authz.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.filter.network; -import "api/grpc_cluster.proto"; +import "api/grpc_service.proto"; import "validate/validate.proto"; @@ -15,7 +15,7 @@ message ExtAuthz { string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; // The external authorization gRPC service configuration. - GrpcCluster grpc_cluster = 2; + GrpcService grpc_service = 2; // The filter's behaviour in case the external authorization service does // not respond back. If set to true then in case of failure to get a diff --git a/api/filter/network/http_connection_manager.proto b/api/filter/network/http_connection_manager.proto index 738dfa91..fc5e487d 100644 --- a/api/filter/network/http_connection_manager.proto +++ b/api/filter/network/http_connection_manager.proto @@ -4,6 +4,7 @@ package envoy.api.v2.filter.network; option go_package = "network"; import "api/base.proto"; +import "api/config_source.proto"; import "api/protocol.proto"; import "api/rds.proto"; import "api/filter/accesslog/accesslog.proto"; diff --git a/api/grpc_cluster.proto b/api/grpc_cluster.proto deleted file mode 100644 index 0f74c4bb..00000000 --- a/api/grpc_cluster.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; - -package envoy.api.v2; - -import "google/protobuf/duration.proto"; - -import "validate/validate.proto"; - -// [#not-implemented-hide:] -// GrpcCluster is used to expose generic gRPC cluster configuration that may -// be used by filters to interface with a gRPC service. -message GrpcCluster { - // The name of the upstream gRPC cluster. - string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; - - // The timeout for the gRPC request. This is the timeout for a specific - // request. - google.protobuf.Duration timeout = 2; -} diff --git a/api/grpc_service.proto b/api/grpc_service.proto new file mode 100644 index 00000000..e93c16f1 --- /dev/null +++ b/api/grpc_service.proto @@ -0,0 +1,81 @@ +syntax = "proto3"; + +package envoy.api.v2; + +import "api/base.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: gRPC services] +// [#proto-status: draft] + +// gRPC service configuration. This is used by :ref:`ApiConfigSource +// ` and filter configurations. +message GrpcService { + message EnvoyGrpc { + // The name of the upstream gRPC cluster. SSL credentials will be supplied + // in the :ref:`Cluster ` :ref:`tls_context + // `. + string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; + } + + message GoogleGrpc { + // The target URI when using the `Google C++ gRPC client + // `_. SSL credentials will be supplied in + // :ref:`credentials `. + string target_uri = 1 [(validate.rules).string.min_bytes = 1]; + + // See https://grpc.io/grpc/cpp/structgrpc_1_1_ssl_credentials_options.html. + message SslCredentials { + // PEM encoded server root certificates. + DataSource root_certs = 1; + + // PEM encoded client private key. + DataSource private_key = 2; + + // PEM encoded client certificate chain. + DataSource cert_chain = 3; + } + SslCredentials ssl_credentials = 2; + } + + oneof target_specifier { + option (validate.required) = true; + + // Envoy's in-built gRPC client. + // See the :ref:`gRPC services overview ` + // documentation for discussion on gRPC client selection. + EnvoyGrpc envoy_grpc = 1; + + // `Google C++ gRPC client `_ + // See the :ref:`gRPC services overview ` + // documentation for discussion on gRPC client selection. + GoogleGrpc google_grpc = 2; + } + + // The timeout for the gRPC request. This is the timeout for a specific + // request. + google.protobuf.Duration timeout = 3; + + // gRPC credentials as described at + // https://grpc.io/docs/guides/auth.html#credential-types. + // + // .. note:: + // + // Credentials are only currently implemented for the Google gRPC client. + message Credentials { + oneof credential_specifier { + option (validate.required) = true; + + // OAuth2 access token, see + // https://grpc.io/grpc/cpp/namespacegrpc.html#ad3a80da696ffdaea943f0f858d7a360d. + string access_token = 1; + // [#comment: TODO(htuch): other gRPC auth types, e.g. IAM credentials, JWT, etc.] + } + } + // A set of credentials that will be composed to form the `channel credentials + // `_. + repeated Credentials credentials = 4; +} diff --git a/api/metrics_service.proto b/api/metrics_service.proto index 43958b3c..edff678e 100644 --- a/api/metrics_service.proto +++ b/api/metrics_service.proto @@ -5,7 +5,7 @@ syntax = "proto3"; package envoy.api.v2; import "api/base.proto"; -import "api/grpc_cluster.proto"; +import "api/grpc_service.proto"; import "metrics.proto"; @@ -41,5 +41,5 @@ message StreamMetricsMessage { // opaque configuration that will be used to create Metrics Service. message MetricsServiceConfig { // The upstream gRPC cluster that hosts the metrics service. - GrpcCluster cluster = 1 [(validate.rules).message.required = true]; + GrpcService grpc_service = 1 [(validate.rules).message.required = true]; } diff --git a/api/sds.proto b/api/sds.proto index 0310f009..148c185e 100644 --- a/api/sds.proto +++ b/api/sds.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.api.v2; import "api/base.proto"; +import "api/config_source.proto"; import "api/discovery.proto"; import "google/api/annotations.proto"; diff --git a/docs/build.sh b/docs/build.sh index e1ec233e..96a5f60e 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -30,8 +30,10 @@ PROTO_RST=" /api/base/api/base.proto.rst /api/bootstrap/api/bootstrap.proto.rst /api/cds/api/cds.proto.rst + /api/config_source/api/config_source.proto.rst /api/discovery/api/discovery.proto.rst /api/eds/api/eds.proto.rst + /api/grpc_service/api/grpc_service.proto.rst /api/health_check/api/health_check.proto.rst /api/lds/api/lds.proto.rst /api/rds/api/rds.proto.rst diff --git a/docs/root/api-v2/api.rst b/docs/root/api-v2/api.rst index 9a5396e5..26611d89 100644 --- a/docs/root/api-v2/api.rst +++ b/docs/root/api-v2/api.rst @@ -8,6 +8,8 @@ v2 API reference :maxdepth: 2 bootstrap.proto + config_source.proto + grpc_service.proto lds.proto cds.proto eds.proto diff --git a/docs/root/intro/arch_overview/grpc.rst b/docs/root/intro/arch_overview/grpc.rst index 4915dd11..ecaf655f 100644 --- a/docs/root/intro/arch_overview/grpc.rst +++ b/docs/root/intro/arch_overview/grpc.rst @@ -24,3 +24,45 @@ application layer: * gRPC-JSON transcoder is supported by a :ref:`filter ` that allows a RESTful JSON API client to send requests to Envoy over HTTP and get proxied to a gRPC service. + +.. _arch_overview_grpc_services: + +gRPC services +------------- + +In addition to proxying gRPC on the data plane, Envoy make use of gRPC for its +control plane, where it :ref:`fetches configuration from management server(s) +` and also in filters, for example for :ref:`rate limiting +` or authorization checks. We refer to these as +*gRPC services*. + +When specifying gRPC services, it's necessary to specify the use of either the +:ref:`Envoy gRPC client ` or the +:ref:`Google C++ gRPC client `. We +discuss the tradeoffs in this choice below. + +The Envoy gRPC client is a minimal custom implementation of gRPC that makes use +of Envoy's HTTP/2 upstream connection management. Services are specified as +regular Envoy :ref:`clusters `, with regular +treatment of :ref:`timeouts, retries `, endpoint +:ref:`discovery `/:ref:`load +balancing/failover `/load reporting, :ref:`circuit +breaking `, :ref:`health checks +`, :ref:`outlier detection +`. They share the same :ref:`connection pooling +` mechanism as the Envoy data plane. Similarly, cluster +:ref:`statistics ` are available for gRPC services. +Since the client is minimal, it does not include advanced gRPC features such as +`OAuth2 `_ or `gRPC-LB +`_ lookaside. + +The Google C++ gRPC client is based on the reference implementation of gRPC +provided by Google at https://github.com/grpc/grpc. It provides advanced gRPC +features that are missing in the Envoy gRPC client. The Google C++ gRPC client +performs its own load balancing, retries, timeouts, endpoint management, etc, +independent of Envoy's cluster management. + +It is recommended to use the Envoy gRPC client in most cases, where the advanced +features in the Google C++ gRPC client are not required. This provides +configuration and monitoring simplicity. Where necessary features are missing +in the Envoy gRPC client, the Google G++ gRPC client should be used instead.