http: prefetch for upstreams (#14143)

Commit Message: Adding predictive prefetch (useful mainly for HTTP/1.1 and TCP proxying) and uncommenting prefetch config.
Additional Description:
Risk Level: low (mostly config guarded)
Testing: unit, integration tests
Docs Changes: APIs unhidden
Release Notes: inline
Fixes #2755

Signed-off-by: Alyssa Wilk <alyssar@chromium.org>

Mirrored from https://github.com/envoyproxy/envoy @ 1c43e39c1366eee1ce241d2963be1d26140462db

envoy/config/cluster/v3/cluster.proto

@@ -583,11 +583,10 @@ message Cluster {
     google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
   }
 
-  // [#not-implemented-hide:]
   message PreconnectPolicy {
     // Indicates how many streams (rounded up) can be anticipated per-upstream for each
     // incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
-    // will only be done if the upstream is healthy.
+    // will only be done if the upstream is healthy and the cluster has traffic.
     //
     // For example if this is 2, for an incoming HTTP/1.1 stream, 2 connections will be
     // established, one for the new incoming stream, and one for a presumed follow-up stream. For
@@ -605,8 +604,7 @@ message Cluster {
     //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight. This means in steady state if a connection is torn down,
-    // a subsequent streams will pay an upstream-rtt latency penalty waiting for streams to be
-    // preconnected.
+    // a subsequent streams will pay an upstream-rtt latency penalty waiting for a new connection.
     //
     // This is limited somewhat arbitrarily to 3 because preconnecting too aggressively can
     // harm latency more than the preconnecting helps.
@@ -616,24 +614,25 @@ message Cluster {
     // Indicates how many many streams (rounded up) can be anticipated across a cluster for each
     // stream, useful for low QPS services. This is currently supported for a subset of
     // deterministic non-hash-based load-balancing algorithms (weighted round robin, random).
-    // Unlike per_upstream_preconnect_ratio this preconnects across the upstream instances in a
+    // Unlike *per_upstream_preconnect_ratio* this preconnects across the upstream instances in a
     // cluster, doing best effort predictions of what upstream would be picked next and
     // pre-establishing a connection.
     //
+    // Preconnecting will be limited to one preconnect per configured upstream in the cluster and will
+    // only be done if there are healthy upstreams and the cluster has traffic.
+    //
     // For example if preconnecting is set to 2 for a round robin HTTP/2 cluster, on the first
     // incoming stream, 2 connections will be preconnected - one to the first upstream for this
     // cluster, one to the second on the assumption there will be a follow-up stream.
     //
-    // Preconnecting will be limited to one preconnect per configured upstream in the cluster.
-    //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight, so during warm up and in steady state if a connection
     // is closed (and per_upstream_preconnect_ratio is not set), there will be a latency hit for
     // connection establishment.
     //
     // If both this and preconnect_ratio are set, Envoy will make sure both predicted needs are met,
-    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each upstream.
-    // TODO(alyssawilk) per LB docs and LB overview docs when unhiding.
+    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each
+    // upstream.
     google.protobuf.DoubleValue predictive_preconnect_ratio = 2
         [(validate.rules).double = {lte: 3.0 gte: 1.0}];
   }
@@ -1028,7 +1027,6 @@ message Cluster {
   // Configuration to track optional cluster stats.
   TrackClusterStats track_cluster_stats = 49;
 
-  // [#not-implemented-hide:]
   // Preconnect configuration for this cluster.
   PreconnectPolicy preconnect_policy = 50;
 

envoy/config/cluster/v4alpha/cluster.proto

@@ -588,14 +588,13 @@ message Cluster {
     google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
   }
 
-  // [#not-implemented-hide:]
   message PreconnectPolicy {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.config.cluster.v3.Cluster.PreconnectPolicy";
 
     // Indicates how many streams (rounded up) can be anticipated per-upstream for each
     // incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
-    // will only be done if the upstream is healthy.
+    // will only be done if the upstream is healthy and the cluster has traffic.
     //
     // For example if this is 2, for an incoming HTTP/1.1 stream, 2 connections will be
     // established, one for the new incoming stream, and one for a presumed follow-up stream. For
@@ -613,8 +612,7 @@ message Cluster {
     //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight. This means in steady state if a connection is torn down,
-    // a subsequent streams will pay an upstream-rtt latency penalty waiting for streams to be
-    // preconnected.
+    // a subsequent streams will pay an upstream-rtt latency penalty waiting for a new connection.
     //
     // This is limited somewhat arbitrarily to 3 because preconnecting too aggressively can
     // harm latency more than the preconnecting helps.
@@ -624,24 +622,25 @@ message Cluster {
     // Indicates how many many streams (rounded up) can be anticipated across a cluster for each
     // stream, useful for low QPS services. This is currently supported for a subset of
     // deterministic non-hash-based load-balancing algorithms (weighted round robin, random).
-    // Unlike per_upstream_preconnect_ratio this preconnects across the upstream instances in a
+    // Unlike *per_upstream_preconnect_ratio* this preconnects across the upstream instances in a
     // cluster, doing best effort predictions of what upstream would be picked next and
     // pre-establishing a connection.
     //
+    // Preconnecting will be limited to one preconnect per configured upstream in the cluster and will
+    // only be done if there are healthy upstreams and the cluster has traffic.
+    //
     // For example if preconnecting is set to 2 for a round robin HTTP/2 cluster, on the first
     // incoming stream, 2 connections will be preconnected - one to the first upstream for this
     // cluster, one to the second on the assumption there will be a follow-up stream.
     //
-    // Preconnecting will be limited to one preconnect per configured upstream in the cluster.
-    //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight, so during warm up and in steady state if a connection
     // is closed (and per_upstream_preconnect_ratio is not set), there will be a latency hit for
     // connection establishment.
     //
     // If both this and preconnect_ratio are set, Envoy will make sure both predicted needs are met,
-    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each upstream.
-    // TODO(alyssawilk) per LB docs and LB overview docs when unhiding.
+    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each
+    // upstream.
     google.protobuf.DoubleValue predictive_preconnect_ratio = 2
         [(validate.rules).double = {lte: 3.0 gte: 1.0}];
   }
@@ -968,7 +967,6 @@ message Cluster {
   // Configuration to track optional cluster stats.
   TrackClusterStats track_cluster_stats = 49;
 
-  // [#not-implemented-hide:]
   // Preconnect configuration for this cluster.
   PreconnectPolicy preconnect_policy = 50;