access log: runtime filter enhancements (#490)

1) Add ability to runtime filter to configure default, divisor, and
   whether independent randomness is used.
2) Also add LE to the comparison filter.

Signed-off-by: Matt Klein <mklein@lyft.com>

STYLE.md

@@ -86,7 +86,12 @@ In addition, the following conventions should be followed:
 
   This is more efficient, extendable and self-describing.
 
-* To represent percentage values, use the Percent message type defined in [api/base.proto](api/base.proto).
+* The API includes two types for representing [percents](envoy/type/percent.proto). `Percent` is
+  effectively a double value in the range 0.0-100.0. `FractionalPercent` is an integral fraction
+  that can be used to create a truncated percentage also in the range 0.0-100.0. In high performance
+  paths, `FractionalPercent` is preferred as randomness calculations can be performed using integral
+  modulo and comparison operations only without any floating point conversions. Typically, most
+  users do not need infinite precision in these paths.
 
 * For enum types, if one of the enum values is used for most cases, make it the
   first enum value with `0` numeric value. Otherwise, define the first enum

docs/BUILD

@@ -50,6 +50,7 @@ proto_library(
         "//envoy/service/discovery/v2:ads",
         "//envoy/service/load_stats/v2:lrs",
         "//envoy/service/metrics/v2:metrics_service",
+        "//envoy/type:percent",
         "//envoy/type:range",
     ],
 )

docs/build.sh

@@ -67,6 +67,7 @@ PROTO_RST="
   /envoy/config/filter/network/rate_limit/v2/rate_limit/envoy/config/filter/network/rate_limit/v2/rate_limit.proto.rst
   /envoy/config/filter/network/redis_proxy/v2/redis_proxy/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto.rst
   /envoy/config/filter/network/tcp_proxy/v2/tcp_proxy/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto.rst
+  /envoy/type/percent/envoy/type/percent.proto.rst
   /envoy/type/range/envoy/type/range.proto.rst
 "
 

docs/root/api-v2/api.rst

@@ -13,4 +13,4 @@ v2 API reference
   http_routes/http_routes
   config/filter/filter
   common_messages/common_messages
-  types/range
+  types/types

docs/root/api-v2/types/types.rst

@@ -1,8 +1,9 @@
 Types
-=========
+=====
 
 .. toctree::
   :glob:
   :maxdepth: 2
 
+  ../type/percent.proto
   ../type/range.proto

envoy/api/v2/BUILD

@@ -68,6 +68,7 @@ api_proto_library(
         "//envoy/api/v2/core:config_source",
         "//envoy/api/v2/core:health_check",
         "//envoy/api/v2/core:protocol",
+        "//envoy/type:percent",
     ],
 )
 
@@ -84,6 +85,7 @@ api_go_grpc_library(
         "//envoy/api/v2/core:config_source_go_proto",
         "//envoy/api/v2/core:health_check_go_proto",
         "//envoy/api/v2/core:protocol_go_proto",
+        "//envoy/type:percent_go_proto",
     ],
 )
 

envoy/api/v2/cds.proto

@@ -13,6 +13,7 @@ import "envoy/api/v2/core/health_check.proto";
 import "envoy/api/v2/core/protocol.proto";
 import "envoy/api/v2/cluster/circuit_breaker.proto";
 import "envoy/api/v2/cluster/outlier_detection.proto";
+import "envoy/type/percent.proto";
 
 import "google/api/annotations.proto";
 import "google/protobuf/duration.proto";
@@ -368,14 +369,14 @@ message Cluster {
     //
     // .. note::
     //   The specified percent will be truncated to the nearest 1%.
-    core.Percent healthy_panic_threshold = 1;
+    envoy.type.Percent healthy_panic_threshold = 1;
     message ZoneAwareLbConfig {
       // [#not-implemented-hide:]
       // Configures percentage of requests that will be considered for zone aware routing
       // if zone aware routing is configured. If not specified, the default is 100%.
       // * :ref:`runtime values <config_cluster_manager_cluster_runtime_zone_routing>`.
       // * :ref:`Zone aware routing support <arch_overview_load_balancing_zone_aware_routing>`.
-      core.Percent routing_enabled = 1;
+      envoy.type.Percent routing_enabled = 1;
       // [#not-implemented-hide:]
       // Configures minimum upstream cluster size required for zone aware routing
       // If upstream cluster size is less than specified, zone aware routing is not performed

envoy/api/v2/core/base.proto

@@ -34,11 +34,6 @@ message Locality {
   string sub_zone = 3;
 }
 
-// Identifies a percentage, in the range [0.0, 100.0].
-message Percent {
-  double value = 1 [(validate.rules).double = {gte: 0, lte: 100}];
-}
-
 // Identifies a specific Envoy instance. The node identifier is presented to the
 // management server, which may use this identifier to distinguish per Envoy
 // configuration for serving.

envoy/config/filter/accesslog/v2/BUILD

@@ -15,5 +15,6 @@ api_proto_library(
         "//envoy/api/v2/core:address",
         "//envoy/api/v2/core:base",
         "//envoy/api/v2/core:grpc_service",
+        "//envoy/type:percent",
     ],
 )

envoy/config/filter/accesslog/v2/accesslog.proto

@@ -5,6 +5,7 @@ option go_package = "v2";
 
 import "envoy/api/v2/core/address.proto";
 import "envoy/api/v2/core/base.proto";
+import "envoy/type/percent.proto";
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
@@ -341,6 +342,9 @@ message ComparisonFilter {
 
     // >=
     GE = 1;
+
+    // <=
+    LE = 2;
   }
 
   // Comparison operator.
@@ -372,16 +376,28 @@ message NotHealthCheckFilter {
 message TraceableFilter {
 }
 
-// Filters for random sampling of requests. Sampling pivots on the header
+// Filters for random sampling of requests.
-// :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being present. If
-// :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` is present, the filter will
-// consistently sample across multiple hosts based on the runtime key value and the value extracted
-// from :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is missing, the
-// filter will randomly sample based on the runtime key value.
 message RuntimeFilter {
-  // Runtime key to get the percentage of requests to be sampled. This runtime
+  // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field.
-  // control is specified in the range 0-100 and defaults to 0.
+  // If found in runtime, this value will replace the default numerator.
   string runtime_key = 1 [(validate.rules).string.min_bytes = 1];
+
+  // The default sampling percentage. If not specified, defaults to 0% with denominator of 100.
+  envoy.type.FractionalPercent percent_sampled = 2;
+
+  // By default, sampling pivots on the header
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` being present. If
+  // :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` is present, the filter will
+  // consistently sample across multiple hosts based on the runtime key value and the value
+  // extracted from :ref:`x-request-id<config_http_conn_man_headers_x-request-id>`. If it is
+  // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based
+  // on the runtime key value alone. *use_independent_randomness* can be used for logging kill
+  // switches within complex nested :ref:`AndFilter
+  // <envoy_api_msg_config.filter.accesslog.v2.AndFilter>` and :ref:`OrFilter
+  // <envoy_api_msg_config.filter.accesslog.v2.OrFilter>` blocks that are easier to reason about
+  // from a probability perspective (i.e., setting to true will cause the filter to behave like
+  // an independent random variable when composed within logical operator filters).
+  bool use_independent_randomness = 3;
 }
 
 // Performs a logical “and” operation on the result of each filter in filters.

envoy/config/filter/http/health_check/v2/BUILD

@@ -6,6 +6,6 @@ api_proto_library(
     name = "health_check",
     srcs = ["health_check.proto"],
     deps = [
-        "//envoy/api/v2/core:base",
+        "//envoy/type:percent",
     ],
 )

envoy/config/filter/http/health_check/v2/health_check.proto

@@ -6,7 +6,7 @@ option go_package = "v2";
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 
-import "envoy/api/v2/core/base.proto";
+import "envoy/type/percent.proto";
 
 import "validate/validate.proto";
 import "gogoproto/gogo.proto";
@@ -29,5 +29,5 @@ message HealthCheck {
   // If operating in non-pass-through mode, specifies a set of upstream cluster
   // names and the minimum percentage of servers in each of those clusters that
   // must be healthy in order for the filter to return a 200.
-  map<string, envoy.api.v2.core.Percent> cluster_min_healthy_percentages = 4;
+  map<string, envoy.type.Percent> cluster_min_healthy_percentages = 4;
 }

envoy/config/filter/network/http_connection_manager/v2/BUILD

@@ -11,5 +11,6 @@ api_proto_library(
         "//envoy/api/v2/core:config_source",
         "//envoy/api/v2/core:protocol",
         "//envoy/config/filter/accesslog/v2:accesslog",
+        "//envoy/type:percent",
     ],
 )

envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto

@@ -3,11 +3,11 @@ syntax = "proto3";
 package envoy.config.filter.network.http_connection_manager.v2;
 option go_package = "v2";
 
-import "envoy/api/v2/core/base.proto";
 import "envoy/api/v2/core/config_source.proto";
 import "envoy/api/v2/core/protocol.proto";
-import "envoy/config/filter/accesslog/v2/accesslog.proto";
 import "envoy/api/v2/rds.proto";
+import "envoy/config/filter/accesslog/v2/accesslog.proto";
+import "envoy/type/percent.proto";
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
@@ -93,14 +93,14 @@ message HttpConnectionManager {
     // 'tracing.client_sampling' in the :ref:`HTTP Connection Manager
     // <config_http_conn_man_runtime>`.
     // Default: 100%
-    envoy.api.v2.core.Percent client_sampling = 3;
+    envoy.type.Percent client_sampling = 3;
 
     // Target percentage of requests managed by this HTTP connection manager that will be randomly
     // selected for trace generation, if not requested by the client or not forced. This field is
     // a direct analog for the runtime variable 'tracing.random_sampling' in the
     // :ref:`HTTP Connection Manager <config_http_conn_man_runtime>`.
     // Default: 100%
-    envoy.api.v2.core.Percent random_sampling = 4;
+    envoy.type.Percent random_sampling = 4;
 
     // Target percentage of requests managed by this HTTP connection manager that will be traced
     // after all other sampling checks have been applied (client-directed, force tracing, random
@@ -110,7 +110,7 @@ message HttpConnectionManager {
     // analog for the runtime variable 'tracing.global_enabled' in the
     // :ref:`HTTP Connection Manager <config_http_conn_man_runtime>`.
     // Default: 100%
-    envoy.api.v2.core.Percent overall_sampling = 5;
+    envoy.type.Percent overall_sampling = 5;
   }
 
   // Presence of the object defines whether the connection manager

envoy/type/BUILD

@@ -2,6 +2,17 @@ load("//bazel:api_build_system.bzl", "api_proto_library", "api_go_proto_library"
 
 licenses(["notice"])  # Apache 2
 
+api_proto_library(
+    name = "percent",
+    srcs = ["percent.proto"],
+    visibility = ["//visibility:public"],
+)
+
+api_go_proto_library(
+    name = "percent",
+    proto = ":percent",
+)
+
 api_proto_library(
     name = "range",
     srcs = ["range.proto"],

envoy/type/percent.proto

@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+package envoy.type;
+
+import "google/protobuf/wrappers.proto";
+
+import "validate/validate.proto";
+
+// [#protodoc-title: Percent]
+
+// Identifies a percentage, in the range [0.0, 100.0].
+message Percent {
+  double value = 1 [(validate.rules).double = {gte: 0, lte: 100}];
+}
+
+// A fractional percentage is used in cases in which for performance reasons performing floating
+// point to integer conversions during randomness calculations is undesirable. The message includes
+// both a numerator and denominator that together determine the final fractional value.
+//
+// * **Example**: 1/100 = 1%.
+// * **Example**: 3/10000 = 0.03%.
+message FractionalPercent {
+  // Specifies the numerator. Defaults to 0.
+  uint32 numerator = 1;
+
+  // Fraction percentages support several fixed denominator values.
+  enum DenominatorType {
+    // 100.
+    //
+    // **Example**: 1/100 = 1%.
+    HUNDRED = 0;
+
+    // 10,000.
+    //
+    // **Example**: 1/10000 = 0.01%.
+    TEN_THOUSAND = 1;
+
+    // 1,000,000.
+    //
+    // **Example**: 1/1000000 = 0.0001%.
+    MILLION = 2;
+  }
+
+  // Specifies the denominator. If the denominator specified is less than the numerator, the final
+  // fractional percentage is capped at 1 (100%).
+  DenominatorType denominator = 2 [(validate.rules).enum.defined_only = true];
+}