From e9c53c404b0105cd42cb4cf015251e5aa9187cd3 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 1 Mar 2018 13:36:33 -0800 Subject: [PATCH] access log: runtime filter enhancements (#490) 1) Add ability to runtime filter to configure default, divisor, and whether independent randomness is used. 2) Also add LE to the comparison filter. Signed-off-by: Matt Klein --- STYLE.md | 7 ++- docs/BUILD | 1 + docs/build.sh | 1 + docs/root/api-v2/api.rst | 2 +- .../api-v2/types/{range.rst => types.rst} | 3 +- envoy/api/v2/BUILD | 2 + envoy/api/v2/cds.proto | 5 +- envoy/api/v2/core/base.proto | 5 -- envoy/config/filter/accesslog/v2/BUILD | 1 + .../filter/accesslog/v2/accesslog.proto | 32 +++++++++---- .../config/filter/http/health_check/v2/BUILD | 2 +- .../http/health_check/v2/health_check.proto | 4 +- .../network/http_connection_manager/v2/BUILD | 1 + .../v2/http_connection_manager.proto | 10 ++-- envoy/type/BUILD | 11 +++++ envoy/type/percent.proto | 47 +++++++++++++++++++ 16 files changed, 108 insertions(+), 26 deletions(-) rename docs/root/api-v2/types/{range.rst => types.rst} (69%) create mode 100644 envoy/type/percent.proto diff --git a/STYLE.md b/STYLE.md index 56c2328c..7c96fefe 100644 --- a/STYLE.md +++ b/STYLE.md @@ -86,7 +86,12 @@ In addition, the following conventions should be followed: This is more efficient, extendable and self-describing. -* To represent percentage values, use the Percent message type defined in [api/base.proto](api/base.proto). +* The API includes two types for representing [percents](envoy/type/percent.proto). `Percent` is + effectively a double value in the range 0.0-100.0. `FractionalPercent` is an integral fraction + that can be used to create a truncated percentage also in the range 0.0-100.0. In high performance + paths, `FractionalPercent` is preferred as randomness calculations can be performed using integral + modulo and comparison operations only without any floating point conversions. Typically, most + users do not need infinite precision in these paths. * For enum types, if one of the enum values is used for most cases, make it the first enum value with `0` numeric value. Otherwise, define the first enum diff --git a/docs/BUILD b/docs/BUILD index 5157d7c5..5bb0beb5 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -50,6 +50,7 @@ proto_library( "//envoy/service/discovery/v2:ads", "//envoy/service/load_stats/v2:lrs", "//envoy/service/metrics/v2:metrics_service", + "//envoy/type:percent", "//envoy/type:range", ], ) diff --git a/docs/build.sh b/docs/build.sh index a41ffcee..e93299ee 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -67,6 +67,7 @@ PROTO_RST=" /envoy/config/filter/network/rate_limit/v2/rate_limit/envoy/config/filter/network/rate_limit/v2/rate_limit.proto.rst /envoy/config/filter/network/redis_proxy/v2/redis_proxy/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto.rst /envoy/config/filter/network/tcp_proxy/v2/tcp_proxy/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto.rst + /envoy/type/percent/envoy/type/percent.proto.rst /envoy/type/range/envoy/type/range.proto.rst " diff --git a/docs/root/api-v2/api.rst b/docs/root/api-v2/api.rst index 9c696075..a4259ad7 100644 --- a/docs/root/api-v2/api.rst +++ b/docs/root/api-v2/api.rst @@ -13,4 +13,4 @@ v2 API reference http_routes/http_routes config/filter/filter common_messages/common_messages - types/range + types/types diff --git a/docs/root/api-v2/types/range.rst b/docs/root/api-v2/types/types.rst similarity index 69% rename from docs/root/api-v2/types/range.rst rename to docs/root/api-v2/types/types.rst index 4372ea8b..116d6c3c 100644 --- a/docs/root/api-v2/types/range.rst +++ b/docs/root/api-v2/types/types.rst @@ -1,8 +1,9 @@ Types -========= +===== .. toctree:: :glob: :maxdepth: 2 + ../type/percent.proto ../type/range.proto diff --git a/envoy/api/v2/BUILD b/envoy/api/v2/BUILD index 70a8141c..10d8ddc2 100644 --- a/envoy/api/v2/BUILD +++ b/envoy/api/v2/BUILD @@ -68,6 +68,7 @@ api_proto_library( "//envoy/api/v2/core:config_source", "//envoy/api/v2/core:health_check", "//envoy/api/v2/core:protocol", + "//envoy/type:percent", ], ) @@ -84,6 +85,7 @@ api_go_grpc_library( "//envoy/api/v2/core:config_source_go_proto", "//envoy/api/v2/core:health_check_go_proto", "//envoy/api/v2/core:protocol_go_proto", + "//envoy/type:percent_go_proto", ], ) diff --git a/envoy/api/v2/cds.proto b/envoy/api/v2/cds.proto index 9be20a86..3df92cf8 100644 --- a/envoy/api/v2/cds.proto +++ b/envoy/api/v2/cds.proto @@ -13,6 +13,7 @@ import "envoy/api/v2/core/health_check.proto"; import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/cluster/circuit_breaker.proto"; import "envoy/api/v2/cluster/outlier_detection.proto"; +import "envoy/type/percent.proto"; import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; @@ -368,14 +369,14 @@ message Cluster { // // .. note:: // The specified percent will be truncated to the nearest 1%. - core.Percent healthy_panic_threshold = 1; + envoy.type.Percent healthy_panic_threshold = 1; message ZoneAwareLbConfig { // [#not-implemented-hide:] // Configures percentage of requests that will be considered for zone aware routing // if zone aware routing is configured. If not specified, the default is 100%. // * :ref:`runtime values `. // * :ref:`Zone aware routing support `. - core.Percent routing_enabled = 1; + envoy.type.Percent routing_enabled = 1; // [#not-implemented-hide:] // Configures minimum upstream cluster size required for zone aware routing // If upstream cluster size is less than specified, zone aware routing is not performed diff --git a/envoy/api/v2/core/base.proto b/envoy/api/v2/core/base.proto index ed93c06d..ade2f025 100644 --- a/envoy/api/v2/core/base.proto +++ b/envoy/api/v2/core/base.proto @@ -34,11 +34,6 @@ message Locality { string sub_zone = 3; } -// Identifies a percentage, in the range [0.0, 100.0]. -message Percent { - double value = 1 [(validate.rules).double = {gte: 0, lte: 100}]; -} - // Identifies a specific Envoy instance. The node identifier is presented to the // management server, which may use this identifier to distinguish per Envoy // configuration for serving. diff --git a/envoy/config/filter/accesslog/v2/BUILD b/envoy/config/filter/accesslog/v2/BUILD index 66298a63..d6f3db9d 100644 --- a/envoy/config/filter/accesslog/v2/BUILD +++ b/envoy/config/filter/accesslog/v2/BUILD @@ -15,5 +15,6 @@ api_proto_library( "//envoy/api/v2/core:address", "//envoy/api/v2/core:base", "//envoy/api/v2/core:grpc_service", + "//envoy/type:percent", ], ) diff --git a/envoy/config/filter/accesslog/v2/accesslog.proto b/envoy/config/filter/accesslog/v2/accesslog.proto index acc7fad0..eecc8dd4 100644 --- a/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/envoy/config/filter/accesslog/v2/accesslog.proto @@ -5,6 +5,7 @@ option go_package = "v2"; import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; +import "envoy/type/percent.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; @@ -341,6 +342,9 @@ message ComparisonFilter { // >= GE = 1; + + // <= + LE = 2; } // Comparison operator. @@ -372,16 +376,28 @@ message NotHealthCheckFilter { message TraceableFilter { } -// Filters for random sampling of requests. Sampling pivots on the header -// :ref:`x-request-id` being present. If -// :ref:`x-request-id` is present, the filter will -// consistently sample across multiple hosts based on the runtime key value and the value extracted -// from :ref:`x-request-id`. If it is missing, the -// filter will randomly sample based on the runtime key value. +// Filters for random sampling of requests. message RuntimeFilter { - // Runtime key to get the percentage of requests to be sampled. This runtime - // control is specified in the range 0-100 and defaults to 0. + // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field. + // If found in runtime, this value will replace the default numerator. string runtime_key = 1 [(validate.rules).string.min_bytes = 1]; + + // The default sampling percentage. If not specified, defaults to 0% with denominator of 100. + envoy.type.FractionalPercent percent_sampled = 2; + + // By default, sampling pivots on the header + // :ref:`x-request-id` being present. If + // :ref:`x-request-id` is present, the filter will + // consistently sample across multiple hosts based on the runtime key value and the value + // extracted from :ref:`x-request-id`. If it is + // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based + // on the runtime key value alone. *use_independent_randomness* can be used for logging kill + // switches within complex nested :ref:`AndFilter + // ` and :ref:`OrFilter + // ` blocks that are easier to reason about + // from a probability perspective (i.e., setting to true will cause the filter to behave like + // an independent random variable when composed within logical operator filters). + bool use_independent_randomness = 3; } // Performs a logical “and” operation on the result of each filter in filters. diff --git a/envoy/config/filter/http/health_check/v2/BUILD b/envoy/config/filter/http/health_check/v2/BUILD index 5ca79e7b..eca07393 100644 --- a/envoy/config/filter/http/health_check/v2/BUILD +++ b/envoy/config/filter/http/health_check/v2/BUILD @@ -6,6 +6,6 @@ api_proto_library( name = "health_check", srcs = ["health_check.proto"], deps = [ - "//envoy/api/v2/core:base", + "//envoy/type:percent", ], ) diff --git a/envoy/config/filter/http/health_check/v2/health_check.proto b/envoy/config/filter/http/health_check/v2/health_check.proto index 1908876c..185a9117 100644 --- a/envoy/config/filter/http/health_check/v2/health_check.proto +++ b/envoy/config/filter/http/health_check/v2/health_check.proto @@ -6,7 +6,7 @@ option go_package = "v2"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; -import "envoy/api/v2/core/base.proto"; +import "envoy/type/percent.proto"; import "validate/validate.proto"; import "gogoproto/gogo.proto"; @@ -29,5 +29,5 @@ message HealthCheck { // If operating in non-pass-through mode, specifies a set of upstream cluster // names and the minimum percentage of servers in each of those clusters that // must be healthy in order for the filter to return a 200. - map cluster_min_healthy_percentages = 4; + map cluster_min_healthy_percentages = 4; } diff --git a/envoy/config/filter/network/http_connection_manager/v2/BUILD b/envoy/config/filter/network/http_connection_manager/v2/BUILD index a1ef916c..0d5205de 100644 --- a/envoy/config/filter/network/http_connection_manager/v2/BUILD +++ b/envoy/config/filter/network/http_connection_manager/v2/BUILD @@ -11,5 +11,6 @@ api_proto_library( "//envoy/api/v2/core:config_source", "//envoy/api/v2/core:protocol", "//envoy/config/filter/accesslog/v2:accesslog", + "//envoy/type:percent", ], ) diff --git a/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 34951e09..5e820bb2 100644 --- a/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -3,11 +3,11 @@ syntax = "proto3"; package envoy.config.filter.network.http_connection_manager.v2; option go_package = "v2"; -import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/core/config_source.proto"; import "envoy/api/v2/core/protocol.proto"; -import "envoy/config/filter/accesslog/v2/accesslog.proto"; import "envoy/api/v2/rds.proto"; +import "envoy/config/filter/accesslog/v2/accesslog.proto"; +import "envoy/type/percent.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; @@ -93,14 +93,14 @@ message HttpConnectionManager { // 'tracing.client_sampling' in the :ref:`HTTP Connection Manager // `. // Default: 100% - envoy.api.v2.core.Percent client_sampling = 3; + envoy.type.Percent client_sampling = 3; // Target percentage of requests managed by this HTTP connection manager that will be randomly // selected for trace generation, if not requested by the client or not forced. This field is // a direct analog for the runtime variable 'tracing.random_sampling' in the // :ref:`HTTP Connection Manager `. // Default: 100% - envoy.api.v2.core.Percent random_sampling = 4; + envoy.type.Percent random_sampling = 4; // Target percentage of requests managed by this HTTP connection manager that will be traced // after all other sampling checks have been applied (client-directed, force tracing, random @@ -110,7 +110,7 @@ message HttpConnectionManager { // analog for the runtime variable 'tracing.global_enabled' in the // :ref:`HTTP Connection Manager `. // Default: 100% - envoy.api.v2.core.Percent overall_sampling = 5; + envoy.type.Percent overall_sampling = 5; } // Presence of the object defines whether the connection manager diff --git a/envoy/type/BUILD b/envoy/type/BUILD index 266131f9..7db404b3 100644 --- a/envoy/type/BUILD +++ b/envoy/type/BUILD @@ -2,6 +2,17 @@ load("//bazel:api_build_system.bzl", "api_proto_library", "api_go_proto_library" licenses(["notice"]) # Apache 2 +api_proto_library( + name = "percent", + srcs = ["percent.proto"], + visibility = ["//visibility:public"], +) + +api_go_proto_library( + name = "percent", + proto = ":percent", +) + api_proto_library( name = "range", srcs = ["range.proto"], diff --git a/envoy/type/percent.proto b/envoy/type/percent.proto new file mode 100644 index 00000000..7b8048e4 --- /dev/null +++ b/envoy/type/percent.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package envoy.type; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Percent] + +// Identifies a percentage, in the range [0.0, 100.0]. +message Percent { + double value = 1 [(validate.rules).double = {gte: 0, lte: 100}]; +} + +// A fractional percentage is used in cases in which for performance reasons performing floating +// point to integer conversions during randomness calculations is undesirable. The message includes +// both a numerator and denominator that together determine the final fractional value. +// +// * **Example**: 1/100 = 1%. +// * **Example**: 3/10000 = 0.03%. +message FractionalPercent { + // Specifies the numerator. Defaults to 0. + uint32 numerator = 1; + + // Fraction percentages support several fixed denominator values. + enum DenominatorType { + // 100. + // + // **Example**: 1/100 = 1%. + HUNDRED = 0; + + // 10,000. + // + // **Example**: 1/10000 = 0.01%. + TEN_THOUSAND = 1; + + // 1,000,000. + // + // **Example**: 1/1000000 = 0.0001%. + MILLION = 2; + } + + // Specifies the denominator. If the denominator specified is less than the numerator, the final + // fractional percentage is capped at 1 (100%). + DenominatorType denominator = 2 [(validate.rules).enum.defined_only = true]; +}